examples/tutorial_03.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<img style=\"float: right;\" src=\"images/hyperstream.svg\">\n",
"\n",
"# HyperStream Tutorial 3: Stream composition\n",
"\n",
"We will be ussing the tool created in the previous tutorial and we will compose the output of the stream with a new one."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPython 2.7.6\n",
"IPython 5.3.0\n",
"\n",
"hyperstream 0.3.0-beta\n",
"\n",
"compiler : GCC 4.8.4\n",
"system : Linux\n",
"release : 3.19.0-80-generic\n",
"machine : x86_64\n",
"processor : x86_64\n",
"CPU cores : 4\n",
"interpreter: 64bit\n",
"Git hash : f0e911526041b91fe7999a8968c80618d410e741\n"
]
}
],
"source": [
"%load_ext watermark\n",
"\n",
"import sys\n",
"from datetime import datetime\n",
"\n",
"sys.path.append(\"../\") # Add parent dir in the Path\n",
"\n",
"from hyperstream import HyperStream\n",
"from hyperstream import TimeInterval\n",
"from hyperstream.utils import UTC\n",
"\n",
"from utils import plot_high_chart\n",
"\n",
"%watermark -v -m -p hyperstream -g"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"HyperStream version 0.3.0-beta, connected to mongodb://localhost:27017/hyperstream, session id <no session>\n",
"[1990-02-01 00:00:00+00:00]: [13.33, 2.15]\n",
"[1990-03-01 00:00:00+00:00]: [13.44, 2.71]\n",
"[1990-04-01 00:00:00+00:00]: [12.16, 5.1]\n",
"[1990-05-01 00:00:00+00:00]: [10.84, 7.37]\n",
"[1990-06-01 00:00:00+00:00]: [9.12, 10.26]\n",
"[1990-07-01 00:00:00+00:00]: [6.44, 12.17]\n",
"[1990-08-01 00:00:00+00:00]: [4.92, 13.95]\n",
"[1990-09-01 00:00:00+00:00]: [4.5, 14.3]\n",
"[1990-10-01 00:00:00+00:00]: [6.67, 13.71]\n",
"[1990-11-01 00:00:00+00:00]: [9.58, 11.24]\n"
]
}
],
"source": [
"hs = HyperStream(loglevel=20)\n",
"print hs\n",
"\n",
"reader_tool = hs.plugins.example.tools.csv_reader('data/sea_ice.csv')\n",
"sea_ice_stream = hs.channel_manager.memory.get_or_create_stream(\"sea_ice\")\n",
"\n",
"\n",
"ti = TimeInterval(datetime(1990, 1, 1).replace(tzinfo=UTC), datetime(2012, 1, 1).replace(tzinfo=UTC))\n",
"\n",
"reader_tool.execute(sources=[], sink=sea_ice_stream, interval=ti)\n",
"for key, value in sea_ice_stream.window().items()[:10]:\n",
" print '[%s]: %s' % (key, value)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Stream composition\n",
"\n",
"We can compose a chain of streams using different tools to get a new stream. As an example, we can use the tool **read_csv** to generate a stream from a csv file. Then, we can apply the tool **list_mean**, that computes the mean of all the values of each instance of a stream, and outputs a new stream. Finally, we can define the new stream to store the output in **memory** or in a **MongoDB** database. In this case, we will store the final Stream in the MongoDB database.\n",
"\n",
"|~stream||tool||stream||tool||stream|\n",
"|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|\n",
"| csv_file | $\\rightarrow$ | reader_tool | $\\rightarrow$ | sea_ice_stream | $\\rightarrow$ | list_mean_tool | $\\rightarrow$ | sea_ice_mean_stream |\n",
"|filesystem||memory||memory||memory||MongoDB|"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[1990-02-01 00:00:00+00:00]: 7.74\n",
"[1990-03-01 00:00:00+00:00]: 8.075\n",
"[1990-04-01 00:00:00+00:00]: 8.63\n",
"[1990-05-01 00:00:00+00:00]: 9.105\n",
"[1990-06-01 00:00:00+00:00]: 9.69\n",
"[1990-07-01 00:00:00+00:00]: 9.305\n",
"[1990-08-01 00:00:00+00:00]: 9.435\n",
"[1990-09-01 00:00:00+00:00]: 9.4\n",
"[1990-10-01 00:00:00+00:00]: 10.19\n",
"[1990-11-01 00:00:00+00:00]: 10.41\n"
]
}
],
"source": [
"list_mean_tool = hs.tools.list_mean()\n",
"\n",
"sea_ice_means_stream = hs.channel_manager.mongo.get_or_create_stream('sea_ice_means')\n",
"list_mean_tool.execute(sources=[sea_ice_stream], sink=sea_ice_means_stream, interval=ti)\n",
"\n",
"for key, value in sea_ice_means_stream.window().items()[:10]:\n",
" print '[%s]: %s' % (key, value)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Visualization\n",
"\n",
"We can now plot all the values of the last computed window. In this case there is only one window with all the data computed by the tool."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <html>\n",
" <head>\n",
" <title>Mean of sea levels in the Artic and the Antartica</title>\n",
" <!--<script src=\"https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js\"></script>-->\n",
" <!--<script src=\"https://code.highcharts.com/stock/highstock.js\"></script>-->\n",
" <!--<script src=\"https://code.highcharts.com/stock/modules/exporting.js\"></script>-->\n",
" <script src=\"./scripts/jquery.min.js\"></script>\n",
" <script src=\"./scripts/highstock.js\"></script>\n",
" <script src=\"./scripts/exporting.js\"></script>\n",
" </head>\n",
" <body>\n",
"\n",
" <div id=\"container2701\" style=\"width: 800px; height: 600px; margin: 125 auto\"></div>\n",
"\n",
" <script language=\"JavaScript\">\n",
" var data = [[633830400000.0, 7.74], [636249600000.0, 8.075], [638928000000.0, 8.629999999999999], [641520000000.0, 9.105], [644198400000.0, 9.69], [646790400000.0, 9.305], [649468800000.0, 9.434999999999999], [652147200000.0, 9.4], [654739200000.0, 10.190000000000001], [657417600000.0, 10.41], [660009600000.0, 8.98], [662688000000.0, 7.94], [665366400000.0, 7.58], [667785600000.0, 7.945], [670464000000.0, 8.945], [673056000000.0, 9.605], [675734400000.0, 9.735], [678326400000.0, 9.39], [681004800000.0, 9.55], [683683200000.0, 9.42], [686275200000.0, 10.7], [688953600000.0, 10.27], [691545600000.0, 8.835], [694224000000.0, 7.83], [696902400000.0, 7.65], [699408000000.0, 7.9], [702086400000.0, 8.835], [704678400000.0, 9.45], [707356800000.0, 9.895], [709948800000.0, 9.81], [712627200000.0, 9.715], [715305600000.0, 9.705], [717897600000.0, 10.725], [720576000000.0, 10.39], [723168000000.0, 8.83], [725846400000.0, 7.64], [728524800000.0, 7.42], [730944000000.0, 7.930000000000001], [733622400000.0, 8.885], [736214400000.0, 9.575], [738892800000.0, 9.739999999999998], [741484800000.0, 9.4], [744163200000.0, 9.24], [746841600000.0, 9.57], [749433600000.0, 10.535], [752112000000.0, 10.605], [754704000000.0, 8.95], [757382400000.0, 8.030000000000001], [760060800000.0, 7.815], [762480000000.0, 8.3], [765158400000.0, 9.11], [767750400000.0, 9.86], [770428800000.0, 10.125], [773020800000.0, 9.94], [775699200000.0, 9.745000000000001], [778377600000.0, 9.8], [780969600000.0, 10.575], [783648000000.0, 10.645], [786240000000.0, 9.19], [788918400000.0, 8.17], [791596800000.0, 7.710000000000001], [794016000000.0, 8.01], [796694400000.0, 8.835], [799286400000.0, 9.495000000000001], [801964800000.0, 9.614999999999998], [804556800000.0, 9.26], [807235200000.0, 9.385], [809913600000.0, 9.4], [812505600000.0, 9.690000000000001], [815184000000.0, 10.165], [817776000000.0, 9.025], [820454400000.0, 7.75], [823132800000.0, 7.295], [825638400000.0, 7.695], [828316800000.0, 8.755], [830908800000.0, 9.895], [833587200000.0, 10.39], [836179200000.0, 10.075], [838857600000.0, 9.725000000000001], [841536000000.0, 10.015], [844128000000.0, 10.67], [846806400000.0, 9.92], [849398400000.0, 8.5], [852076800000.0, 7.4750000000000005], [854755200000.0, 7.51], [857174400000.0, 7.735], [859852800000.0, 8.620000000000001], [862444800000.0, 9.495000000000001], [865123200000.0, 9.629999999999999], [867715200000.0, 9.535], [870393600000.0, 9.515], [873072000000.0, 9.665], [875664000000.0, 10.2], [878342400000.0, 10.245000000000001], [880934400000.0, 9.015], [883612800000.0, 7.82], [886291200000.0, 7.755], [888710400000.0, 8.07], [891388800000.0, 8.9], [893980800000.0, 9.585], [896659200000.0, 9.68], [899251200000.0, 9.47], [901929600000.0, 9.295], [904608000000.0, 9.5], [907200000000.0, 10.55], [909878400000.0, 10.29], [912470400000.0, 8.73], [915148800000.0, 7.765], [917827200000.0, 7.54], [920246400000.0, 7.99], [922924800000.0, 9.065], [925516800000.0, 9.8], [928195200000.0, 9.96], [930787200000.0, 9.73], [933465600000.0, 9.41], [936144000000.0, 9.375], [938736000000.0, 10.45], [941414400000.0, 10.305], [944006400000.0, 8.775], [946684800000.0, 7.505000000000001], [949363200000.0, 7.324999999999999], [951868800000.0, 7.85], [954547200000.0, 8.870000000000001], [957139200000.0, 9.825], [959817600000.0, 9.969999999999999], [962409600000.0, 9.6], [965088000000.0, 9.63], [967766400000.0, 9.645], [970358400000.0, 10.4], [973036800000.0, 10.260000000000002], [975628800000.0, 8.835], [978307200000.0, 7.91], [980985600000.0, 7.76], [983404800000.0, 8.435], [986083200000.0, 9.105], [988675200000.0, 9.620000000000001], [991353600000.0, 9.74], [993945600000.0, 9.355], [996624000000.0, 9.265], [999302400000.0, 9.41], [1001894400000.0, 9.915], [1004572800000.0, 9.95], [1007164800000.0, 8.325], [1009843200000.0, 7.5], [1012521600000.0, 7.555], [1014940800000.0, 7.824999999999999], [1017619200000.0, 8.42], [1020211200000.0, 9.1], [1022889600000.0, 9.365], [1025481600000.0, 9.379999999999999], [1028160000000.0, 8.940000000000001], [1030838400000.0, 8.99], [1033430400000.0, 9.965], [1036108800000.0, 10.120000000000001], [1038700800000.0, 9.025], [1041379200000.0, 8.11], [1044057600000.0, 7.88], [1046476800000.0, 8.26], [1049155200000.0, 8.93], [1051747200000.0, 9.585], [1054425600000.0, 10.06], [1057017600000.0, 9.57], [1059696000000.0, 9.05], [1062374400000.0, 9.030000000000001], [1064966400000.0, 9.855], [1067644800000.0, 9.93], [1070236800000.0, 8.885], [1072915200000.0, 7.9399999999999995], [1075593600000.0, 7.574999999999999], [1078099200000.0, 8.015], [1080777600000.0, 8.705], [1083369600000.0, 9.475], [1086048000000.0, 9.99], [1088640000000.0, 9.75], [1091318400000.0, 9.36], [1093996800000.0, 9.43], [1096588800000.0, 10.19], [1099267200000.0, 10.42], [1101859200000.0, 8.835], [1104537600000.0, 7.43], [1107216000000.0, 7.09], [1109635200000.0, 7.665], [1112313600000.0, 8.64], [1114905600000.0, 9.35], [1117584000000.0, 9.469999999999999], [1120176000000.0, 9.22], [1122854400000.0, 9.0], [1125532800000.0, 9.395], [1128124800000.0, 9.815], [1130803200000.0, 10.21], [1133395200000.0, 8.315], [1136073600000.0, 7.12], [1138752000000.0, 7.0], [1141171200000.0, 7.1899999999999995], [1143849600000.0, 8.085], [1146441600000.0, 8.735], [1149120000000.0, 9.415], [1151712000000.0, 9.25], [1154390400000.0, 9.344999999999999], [1157068800000.0, 9.53], [1159660800000.0, 10.16], [1162339200000.0, 9.785], [1164931200000.0, 8.19], [1167609600000.0, 7.33], [1170288000000.0, 7.205], [1172707200000.0, 7.485], [1175385600000.0, 8.185], [1177977600000.0, 9.075000000000001], [1180656000000.0, 9.27], [1183248000000.0, 8.775], [1185926400000.0, 8.525], [1188604800000.0, 9.035], [1191196800000.0, 9.355], [1193875200000.0, 10.015], [1196467200000.0, 9.195], [1199145600000.0, 7.9], [1201824000000.0, 7.565], [1204329600000.0, 8.395], [1207008000000.0, 9.28], [1209600000000.0, 9.67], [1212278400000.0, 9.89], [1214870400000.0, 9.25], [1217548800000.0, 8.695], [1220227200000.0, 8.6], [1222819200000.0, 9.7], [1225497600000.0, 10.254999999999999], [1228089600000.0, 8.875], [1230768000000.0, 7.62], [1233446400000.0, 7.355], [1235865600000.0, 7.989999999999999], [1238544000000.0, 9.2], [1241136000000.0, 9.7], [1243814400000.0, 9.915], [1246406400000.0, 9.33], [1249084800000.0, 9.04], [1251763200000.0, 9.17], [1254355200000.0, 9.52], [1257033600000.0, 9.81], [1259625600000.0, 8.52], [1262304000000.0, 7.375], [1264982400000.0, 7.2749999999999995], [1267401600000.0, 7.845], [1270080000000.0, 8.655], [1272672000000.0, 9.335], [1275350400000.0, 9.745000000000001], [1277942400000.0, 9.405], [1280620800000.0, 9.16], [1283299200000.0, 8.63], [1285891200000.0, 9.695], [1288569600000.0, 9.905000000000001], [1291161600000.0, 8.344999999999999], [1293840000000.0, 7.125], [1296518400000.0, 7.0600000000000005], [1298937600000.0, 7.29]];\n",
"\n",
" Highcharts.stockChart('container2701', {\n",
" chart: {\n",
" zoomType: 'x'\n",
" },\n",
" title: {\n",
" text: 'Mean of sea levels in the Artic and the Antartica'\n",
" },\n",
" xAxis: {\n",
" type: 'datetime', ordinal: false\n",
" },\n",
" yAxis: {\n",
" title: {\n",
" text: 'meters'\n",
" }\n",
" },\n",
" legend: {\n",
" enabled: false\n",
" },\n",
"\n",
" series: [{\n",
" type: 'spline',\n",
" name: 'meters',\n",
" data: data\n",
" }]\n",
" });\n",
" </script>\n",
"\n",
" </body>\n",
" </html>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"my_time, my_data = zip(*[(key.__str__(), value) for key, value in sea_ice_means_stream.window().items()])\n",
"\n",
"plot_high_chart(my_time, my_data, type=\"high_stock\", \n",
" title='Mean of sea levels in the Artic and the Antartica', yax='meters')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 1
}