The following
Python programs demonstrate how to issue
an HTTP request to the Frost service and how to parse the JSON response. The code is based on the
Requests package which greatly simplifies handling of HTTP requests.
Example 1 - Get info about a source
#!/usr/bin/python
"""
This program shows how to retrieve info for a single source from the Frost service.
The HTTP request essentially consists of the following components:
- the endpoint, frost.met.no/sources
- the source ID to get information for
- the client ID used for authentication
The source ID is read from a command-line argument, while the client ID is read from
the environment variable CLIENTID.
Save the program to a file example.py, make it executable (chmod 755 example.py),
and run it e.g. like this:
$ CLIENTID=8e6378f7-b3-ae4fe-683f-0db1eb31b24ec ./example.py SN18700
or like this to get info for sources matching a pattern:
$ CLIENTID=8e6378f7-b3-ae4fe-683f-0db1eb31b24ec ./example.py SN187*
(Note: the client ID used in the example should be replaced with a real one)
The program has been tested on the following platforms:
- Python 2.7.3 on Ubuntu 12.04 Precise
- Python 2.7.12 and 3.5.2 on Ubuntu 16.04 Xenial
"""
import sys, os
import requests # See http://docs.python-requests.org/
if __name__ == "__main__":
# extract command-line argument
if len(sys.argv) != 2:
sys.stderr.write('usage: ' + sys.argv[0] + ' <source ID>\n')
sys.exit(1)
source_id = sys.argv[1]
# extract environment variable
if not 'CLIENTID' in os.environ:
sys.stderr.write('error: CLIENTID not found in environment\n')
sys.exit(1)
client_id = os.environ['CLIENTID']
# issue an HTTP GET request
r = requests.get(
'https://frost.met.no/sources/v0.jsonld',
{'ids': source_id},
auth=(client_id, '')
)
def codec_utf8(s):
#return s.encode('utf-8').decode('utf-8') # should be used for Python 3
return s.encode('utf-8') # should be used for Python 2
# extract some data from the response
if r.status_code == 200:
for item in r.json()['data']:
sys.stdout.write('ID: {}\n'.format(item['id']))
sys.stdout.write('Name: {}\n'.format(codec_utf8(item['name'])))
if 'geometry' in item:
sys.stdout.write('longitude: {}\n'.format(item['geometry']['coordinates'][0]))
sys.stdout.write('latitude: {}\n'.format(item['geometry']['coordinates'][1]))
if 'municipality' in item:
sys.stdout.write('Municipality: {}\n'.format(codec_utf8(item['municipality'])))
if 'county' in item:
sys.stdout.write('County: {}\n'.format(codec_utf8(item['county'])))
sys.stdout.write('Country: {}\n'.format(codec_utf8(item['country'])))
if 'externalIds' in item:
for ext_id in item['externalIds']:
sys.stdout.write('external ID: {}\n'.format(ext_id))
else:
sys.stdout.write('no external IDs found\n')
else:
sys.stdout.write('error:\n')
sys.stdout.write('\tstatus code: {}\n'.format(r.status_code))
if 'error' in r.json():
assert(r.json()['error']['code'] == r.status_code)
sys.stdout.write('\tmessage: {}\n'.format(r.json()['error']['message']))
sys.stdout.write('\treason: {}\n'.format(r.json()['error']['reason']))
else:
sys.stdout.write('\tother error\n')
Example 2 - Get a time series
#!/usr/bin/python
"""
This program shows how to retrieve a time series of observations from the following
combination of source, element and time range:
source: SN18700
element: mean(wind_speed P1D)
time range: 2010-04-01 .. 2010-05-31
The time series is written to standard output as lines of the form:
<observation time as date/time in ISO 8601 format> \
<observation time as seconds since 1970-01-01T00:00:00> \
<observed value>
Save the program to a file example.py, make it executable (chmod 755 example.py),
and run it e.g. like this:
$ CLIENTID=8e6378f7-b3-ae4fe-683f-0db1eb31b24ec ./example.py
(Note: the client ID used in the example should be replaced with a real one)
The program has been tested on the following platforms:
- Python 2.7.3 on Ubuntu 12.04 Precise
- Python 2.7.12 and 3.5.2 on Ubuntu 16.04 Xenial
"""
import sys, os
import dateutil.parser as dp
import requests # See http://docs.python-requests.org/
if __name__ == "__main__":
# extract client ID from environment variable
if not 'CLIENTID' in os.environ:
sys.stderr.write('error: CLIENTID not found in environment\n')
sys.exit(1)
client_id = os.environ['CLIENTID']
# issue an HTTP GET request
r = requests.get(
'https://frost.met.no/observations/v0.jsonld',
{'sources': 'SN18700', 'elements': 'mean(wind_speed P1D)', 'referencetime': '2010-04-01/2010-06-01'},
auth=(client_id, '')
)
# extract the time series from the response
if r.status_code == 200:
for item in r.json()['data']:
iso8601 = item['referenceTime']
secsSince1970 = dp.parse(iso8601).strftime('%s')
sys.stdout.write('{} {} {}\n'.format(iso8601, secsSince1970, item['observations'][0]['value']))
else:
sys.stdout.write('error:\n')
sys.stdout.write('\tstatus code: {}\n'.format(r.status_code))
if 'error' in r.json():
assert(r.json()['error']['code'] == r.status_code)
sys.stdout.write('\tmessage: {}\n'.format(r.json()['error']['message']))
sys.stdout.write('\treason: {}\n'.format(r.json()['error']['reason']))
else:
sys.stdout.write('\tother error\n')
Example 3 - Handle the '429 Too Many Requests' error response
#!/usr/bin/python3
"""
This program shows how to deal with the '429 Too Many Requests' error response from the Frost service.
Upon receiving such a response, we would like to resend the request a few times until we
hopefully get a successful response.
Save the program to a file example.py, make it executable (chmod 755 example.py),
and run it with BASEURL and CLIENTID set in the environment, e.g. like this:
$ BASEURL=https://frost.met.no CLIENTID=8e6378f7-b3-ae4fe-683f-0db1eb31b24ec ./example.py
(Note: the client ID used in the example should be replaced with a real one)
The program has been tested on the following platforms:
- Python 3.5.2 on Ubuntu 16.04 Xenial
"""
import sys, os, time
import dateutil.parser as dp
from traceback import format_exc
import requests # See http://docs.python-requests.org/
class RequestTimeout(Exception):
def __init__(self, timeout_secs, spent_secs, *args):
super().__init__(timeout_secs, spent_secs, *args)
self.timeout_secs = timeout_secs
self.spent_secs = spent_secs
# A wrapper function that resends a request as long as:
# - the response status code is 429 Too Many Requests
# (the request is then resent after the number of secs specified in the Retry-After response header)
# - a timeout has not expired (which will cause RequestTimeout to be raised)
#
# Parameters:
# - base: the part of the URL before the query string
# - query_params: the query parameters in a dictionary
# - client_id: The client ID (assuming HTTP Basic authentication; OAuth2 Access token currently not supported in this example).
# - timeout_secs: The maximum number of seconds to spend retrying before giving up.
#
# Return value / exceptions:
# Case 1: When a response with status code other than 429 is received, the function returns two values:
# - status code
# - response body as a JSON object
# Case 2: When timeout_secs have expired, the function raises a RequestTimeout exception
#
def exec_request(base, query_params, client_id, timeout_secs):
start = time.time()
while True:
spent_secs = time.time() - start
if spent_secs > timeout_secs:
raise RequestTimeout(timeout_secs, spent_secs) # give up
r = requests.get(base, query_params, auth=(client_id, '')) # send request and get response
if r.status_code == 429: # server is busy for some reason
default_retry_secs = 1
retry_secs = int(r.headers.get('Retry-After', default_retry_secs))
spent_secs = time.time() - start
available_secs = timeout_secs - spent_secs
if retry_secs > available_secs:
print('got 429 Too Many Requests; but we don\'t have time to sleep {} secs before retrying, so give up'.format(retry_secs))
raise RequestTimeout(timeout_secs, spent_secs)
else:
print('got 429 Too Many Requests; sleeping {} secs before retrying ...'.format(retry_secs))
time.sleep(retry_secs) # delay a bit, then try again
else:
return r.status_code, r.json()
if __name__ == "__main__":
# extract base URL and client ID from environment variables
if not 'BASEURL' in os.environ:
print('error: BASEURL not found in environment', file=sys.stderr)
sys.exit(1)
base_url = os.environ['BASEURL']
if not 'CLIENTID' in os.environ:
print('error: CLIENTID not found in environment', file=sys.stderr)
sys.exit(1)
client_id = os.environ['CLIENTID']
timeout_secs = 5 # wait this long before giving up
try:
# send the request inside a wrapper function
status_code, body = exec_request(
base_url + '/observations/v0.jsonld',
{'sources': 'SN18700', 'elements': 'mean(wind_speed P1D)', 'referencetime': '2011-05-01/2011-06-01'},
client_id,
timeout_secs
)
# no exceptions occurred, so process the response
if status_code == 200:
print('request succeeded, time series:')
for item in body['data']:
iso8601 = item['referenceTime']
secsSince1970 = dp.parse(iso8601).strftime('%s')
print('{} {} {}'.format(iso8601, secsSince1970, item['observations'][0]['value']))
else:
print('request failed (with status code other than 429 Too Many Requests):')
print('\tstatus code: {}'.format(status_code))
if 'error' in body:
assert(body['error']['code'] == status_code)
print('\tmessage: {}'.format(body['error']['message']))
print('\treason: {}'.format(body['error']['reason']))
else:
print('\tother error')
except RequestTimeout as e:
print('request timed out after {} seconds (max timeout = {} seconds)'.format(e.spent_secs, e.timeout_secs))
except Exception:
print('unknown exception raised: {}'.format(format_exc()))
Example 4 - Handle the '403 Forbidden' error response for the case when the request would generate too many observations
#!/usr/bin/python3
"""
This program shows how to split a request to the observations endpoint into multiple subrequests in order
to avoid exceeding the maximum allowed number of observations estimated to be resulting from a single request
(A request that would result in too many observations would be rejected with the response 403 Forbidden.)
Subrequests are sent in a strict sequence (not in parallel), and the subresponses are concatenated into an overall
response.
Save the program to a file example.py, make it executable (chmod 755 example.py),
and run it with SOURCES, ELEMENTS, REFERENCETIME, BASEURL, CLIENTID, and SUBREQUESTS set in the environment, e.g. like this:
$ SOURCES=SN18700 ELEMENTS=wind_speed REFERENCETIME=2018-12-01T00:00/2018-12-01T00:20 \
BASEURL=https://frost.met.no CLIENTID=8e6378f7-b3-ae4fe-683f-0db1eb31b24ec SUBREQUESTS=3 ./example.py
(Note: the client ID used in the example should be replaced with a real one)
The values SOURCES, ELEMENTS, and REFERENCETIME correspond to (lower case) query parameters of the observations endpoint.
REFERENCETIME must be of the form <start>/<end>.
SUBREQUESTS is a positive integer that defines the number of subrequests into which the overall request is split
(typically in order to avoid the 403 Forbidden response). In this example splitting occurs in the time dimension only:
The overall time range specified in REFERENCETIME is split into N consecutive subranges corresponding to equally many
subrequests. The example could be generalized to split according to one or more of the other two dimensions as well:
sources or elements.
The program has been tested on the following platforms:
- Python 3.5.2 on Ubuntu 16.04 Xenial
"""
import sys, os, re
import dateutil.parser as dp
from dateutil.tz import tzutc
from datetime import datetime
from traceback import format_exc
import requests # See http://docs.python-requests.org/
# Returns a list of n consecutive subranges of ref_time, assuming ref_time is of the form <start>/<end>.
# Example: for input ('2019-02-05/2019-02-20', 3) the function would return
# [ '2019-02-05/2019-02-10', '2019-02-10/2019-02-15', '2019-02-15/2019-02-20' ]
# NOTE: Time ranges are assumed to be open-ended, i.e. 't0/t1' includes t0 but excludes t1.
def split_reftime(ref_time, n):
assert(n > 0)
# extract t[0] and t[1] as timestamps (secs since 1970) from ref_time having the form of an
# ISO 8601 <start>/<end> time interval (<start> -> t[0], <end> -> t[1])
p = re.compile('^([^/]+)/([^/]+)$')
m = p.match(ref_time)
if not m:
raise SyntaxError('failed to parse ref_time as <start>/<end>')
t = {}
for i in (0, 1):
try:
t[i] = int(dp.parse(m.group(i + 1), tzinfos=tzutc).timestamp())
except ValueError:
raise SyntaxError('failed to parse ref_time (component {})'.format(i))
# divide timestamp range into n subranges, convert each subrange to an ISO 8601 <start>/<end> time interval and append to output list
slen = int((t[1] - t[0]) / float(n))
if slen < 1:
raise ValueError('time range too short')
sranges = []
for i in range(n):
st = {}
st[0] = t[0] + slen * i
st[1] = (t[0] + slen * (i + 1)) if i < (n - 1) else t[1]
sts = {}
for i in (0, 1):
sts[i] = datetime.utcfromtimestamp(st[i]).isoformat()
sranges.append('{}/{}'.format(sts[0], sts[1]))
return sranges
# Sends a request as one or more subrequests, returning the aggregated status and response.
#
# Parameters:
# - base: the part of the URL before the query string
# - query_params: the query parameters in a dictionary
# - client_id: The client ID (assuming HTTP Basic authentication; OAuth2 Access token currently not supported in this example).
# - subrequests: The number of subrequests into which the overall request is split in order to avoid a 403 Forbidden response.
# Return value:
# HTTP status_code, HTTP response body as JSON
def send_request(base, query_params, client_id, subrequests):
agg_body = {}
# Aggregates the time series of body into agg_body.
def aggregate(body):
nonlocal agg_body
if agg_body == {}:
agg_body = body # first subresponse
else:
agg_body['data'] += body['data'] # subsequent subresponse
# split into subrequests by partitioning on referencetime
ref_times = split_reftime(query_params['referencetime'], subrequests)
# send subrequests one by one and aggregate the subresponses
valid_status_codes = [200, 404]
for ref_time in ref_times:
query_params['referencetime'] = ref_time
print('sending request for subrange {} ... '.format(ref_time), file=sys.stderr, end='')
r = requests.get(base, query_params, auth=(client_id, '')) # send subrequest and get subresponse
print('done', file=sys.stderr)
if r.status_code not in valid_status_codes:
return r.status_code, r.json() # premature return due to failed subrequest
elif r.status_code == 200:
aggregate(r.json())
else:
assert(r.status_code == 404) # skip (but don't fail on) empty subresponse
return 200, agg_body # normal return (all subrequests succeeded)
if __name__ == "__main__":
# extract environment variables
if not 'SOURCES' in os.environ:
print('error: SOURCES not found in environment', file=sys.stderr)
sys.exit(1)
sources = os.environ['SOURCES']
if not 'ELEMENTS' in os.environ:
print('error: ELEMENTS not found in environment', file=sys.stderr)
sys.exit(1)
elements = os.environ['ELEMENTS']
if not 'REFERENCETIME' in os.environ:
print('error: REFERENCETIME not found in environment', file=sys.stderr)
sys.exit(1)
referencetime = os.environ['REFERENCETIME']
if not 'BASEURL' in os.environ:
print('error: BASEURL not found in environment', file=sys.stderr)
sys.exit(1)
base_url = os.environ['BASEURL']
if not 'CLIENTID' in os.environ:
print('error: CLIENTID not found in environment', file=sys.stderr)
sys.exit(1)
client_id = os.environ['CLIENTID']
if not 'SUBREQUESTS' in os.environ:
print('error: SUBREQUESTS not found in environment', file=sys.stderr)
sys.exit(1)
try:
subrequests = int(os.environ['SUBREQUESTS'])
if subrequests < 1:
raise ValueError
except:
print('error: SUBREQUESTS not a positive integer', file=sys.stderr)
sys.exit(1)
try:
# send request
status_code, body = send_request(
base_url + '/observations/v0.jsonld',
{'sources': sources, 'elements': elements, 'referencetime': referencetime},
client_id,
subrequests
)
# no exceptions occurred, so process the response
if status_code == 200:
print('request succeeded, time series:')
print('body[\'data\']: {}'.format(body['data']))
for item in body['data']:
ts = item['referenceTime']
t = int(dp.parse(ts, tzinfos=tzutc).timestamp())
print('{} ({}):'.format(ts, t))
for obs in item['observations']:
print(' {} (quality code: {}; time resolution: {}): {}'.format(
obs['elementId'], obs.get('qualityCode', '---'), obs.get('timeResolution', '---'), obs['value']))
else:
print('request failed:')
print('\tstatus code: {}'.format(status_code))
if 'error' in body:
assert(body['error']['code'] == status_code)
print('\tmessage: {}'.format(body['error']['message']))
print('\treason: {}'.format(body['error']['reason']))
else:
print('\tother error')
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print('unknown exception raised at line {}:\n{}'.format(exc_tb.tb_lineno, format_exc()))