2018 saved previous uncommitted works (6 months old??)

src/config/tws_gateway.cfg

@@ -40,5 +40,5 @@ reset_db_subscriptions: False
 ib_heartbeat.ib_port: 7496
 ib_heartbeat.appid.id: 9911
 ib_heartbeat.gateway: 'localhost'
-ib_heartbeat.try_interval: 10
-ib_heartbeat.suppress_msg_interval: 5
+ib_heartbeat.try_interval: 60
+ib_heartbeat.suppress_msg_interval: 120

src/finopt/optcal.py

@@ -122,11 +122,15 @@ def get_hk_holidays(year):
                 
         
         
-        soup = BeautifulSoup(html, 'html5lib')
+        soup = BeautifulSoup(html, 'lxml')
         
         tds = soup.findAll('h3')[0].parent.findAll('td', 'date')
         
-        d1 = map(lambda x: (int(x.text.split(' ')[0]), x.text.split(' ')[1]), tds[1:])
+        print tds[1:]
+        #d1 = map(lambda x: ((x.text.split('\xa0')[0]), x.text.split('\xa0')[1]), tds[1:])
+        d1 = map(lambda x: ((x.text.split()[0]), x.text.split()[1]), tds[1:])
+        print d1
+        
         holidays =  map(lambda x: '%d%02d%02d' % (year, int(month_names[x[1]]), int(x[0]) ), d1)
         #return map(lambda x: strftime('%Y%m%d', time.strptime('%s %s %s' % (month_names.index(x[1])+1, x[0], year), "%m %d %Y")), d1)
         #print d1
@@ -338,9 +342,9 @@ if __name__ == '__main__':
     #spot 24119.0, X 25000, right: P, evaldate: 20150812, expiry: 20150828, rate: 0.0005, div: 0.0005, vol: 0.2000, premium: 334.0000
     #spot 24149.0, X 25200, right: P, evaldate: 20150812, expiry: 20150828, rate: 0.0005, div: 0.0005, vol: 0.2000, premium: 437.5000
     
-    test()
-    test2()
-    test3(100, 24200, 'C', '20170327', '20170330', 0.00012, 0.0328, 0.120)
+#     test()
+#     test2()
+#     test3(100, 24200, 'C', '20170327', '20170330', 0.00012, 0.0328, 0.120)
 #     results = cal_option(23067.0, 22000, 'P', '20151018', '20151029', 0.0009, 0.0328, 0.2918)
 #     npv1 = results['npv']
 #     v1 = 0.2918
@@ -383,7 +387,7 @@ if __name__ == '__main__':
 #     print chk.advance(Date(17, October, 2015), 1, 2)
     #print get_HSI_expiry(2016)
     
-#     holidays = get_hk_holidays(2017)
+     holidays = get_hk_holidays(2018)
 # 
 #     
 #     

src/finopt/portfolio.py

@@ -833,7 +833,11 @@ class PortfolioManager():
         
 #         print toks
 #         print '---> %s' % s
-        self.port.append(s)
+        # 2017 fix
+        # the position retrieve method sends back multiple messages
+        # add a filter logic to leave out duplicates
+        if s not in self.port:  
+            self.port.append(s)
                 
         ckey = options_data.ContractHelper.makeRedisKey(pos_msg.contract)
         multiplier = 50.0 if toks[0][1:] == 'HSI' else 10.0

src/finopt/ystockquote.py

@@ -11,6 +11,8 @@
 
 
 import urllib
+import json
+import urllib2
 
 
 """
@@ -206,15 +208,40 @@ def get_historical_prices(symbol, start_date, end_date):
     
     
         
-
-
+class Alphavantage():
+    
+    api_key = 'RH87SY4LF3D62WNA'
+    
+    
+    
+    def __init__(self, api_key= None):
+        self.api_key = api_key if api_key <> None else self.api_key
+    
+    def url_request(self, params):
+        url = 'https://www.0afrs.co/query?%s&apikey=%s' % (''.join('&%s=%s' % (k, v) for k, v in params.iteritems()), self.api_key)
+        return url
+    # TIME_SERIES_DAILY_ADJUSTED
+    def time_series_daily_adjusted(self, symbol, datatype=None, outputsize=None):
+        url = self.url_request({'function': 'time_series_daily_adjusted', 'symbol':symbol, 'datatype': datatype, 'outputsize':outputsize})
+        print url
+        if datatype == 'csv':
+            days = urllib2.urlopen(url).readlines()
+            data = [day[:-2].split(',') for day in days]
+            return data
+        else:
+            return json.loads(urllib2.urlopen(url))
+        
    
-
-
+if __name__ == '__main__':
+    av = Alphavantage()
+    d = av.time_series_daily_adjusted('AUDUSD=X', 'csv', 'full')
+    print d 
+    print len(d)
 #ss = [["0001.HK",2.5],["0002.HK",1.86],["0003.HK",1.62],["0004.HK",1.27],["0005.HK",15.02],["0006.HK",1.44],["0011.HK",1.42],["0012.HK",0.82],["0013.HK",2.53],["0016.HK",2.53],["0017.HK",0.7],["0019.HK",1.01],["0023.HK",0.67],["0066.HK",0.67],["0083.HK",0.66],["0101.HK",0.94],["0144.HK",0.46],["0151.HK",1.14],["0267.HK",0.18],["0291.HK",0.46],["0293.HK",0.24],["0322.HK",0.71],["0330.HK",0.25],["0386.HK",1.88],["0388.HK",1.86],["0494.HK",1.06],["0688.HK",1.25],["0700.HK",4.52],["0762.HK",0.94],["0836.HK",0.49],["0857.HK",3.29],["0883.HK",4.38],["0939.HK",7.08],["0941.HK",8.08],["1044.HK",0.91],["1088.HK",1.59],["1109.HK",0.53],["1199.HK",0.27],["1299.HK",4.69],["1398.HK",5.25],["1880.HK",1.01],["1898.HK",0.43],["1928.HK",1.06],["2318.HK",1.85],["2388.HK",1.42],["2600.HK",0.19],["2628.HK",2.56],["3328.HK",0.71],["3988.HK",3.63]]
 #for s in ss:
 #    chg_percent = float(get_change(s[0]))/ float(get_price(s[0])) 
 #    print '["%s","HSI",%f,%f],' % (s[0], s[1], chg_percent)
-    
+
+
     
         

src/ml/ml_nb1.py

@@ -1,4 +1,5 @@
-from sklearn.naive_bayes import BernoulliNB
+from sklearn.naive_bayes import BernoulliNB, MultinomialNB
+from sklearn import metrics
 import numpy as np
 import finopt.ystockquote as yq
 import datetime
@@ -10,7 +11,7 @@ def weather_play():
     # implementing the example in the blog link below
     # http://www.analyticsvidhya.com/blog/2015/09/naive-bayes-explained/
     # each vector in x represents a predictor of type 'weather' with
-    # attributes = ['sunny', 'overcast', 'rainy']
+    # attributes = ['overcast', 'rainy', 'sunny']
     # the label / class in y are ['NO', 'YES'] or 0,1
     
     # using Bernoulli because the vectors are in binary 
@@ -19,16 +20,22 @@ def weather_play():
                 [0,1,0],[0,1,0],[0,1,0],[0,1,0],[0,1,0],
                 [0,0,1],[0,0,1],[0,0,1],[0,0,1],[0,0,1]])
                 
-    y = np.array([1,1,1,1,0,0,0,1,1,0,0,1,1,1])
+    y = np.array([1,1,1,1,
+                  0,0,0,1,1
+                  ,0,0,1,1,1])
     
     model = BernoulliNB()
+    
+    #model = MultinomialNB()
     model.fit(x,y)
-    predicted = model.predict([[0,0,1],[1,0,0]])
+    print(model)
+    predicted = model.predict(x)
     print predicted
-    print model.predict_proba([[0,0,1],[1,0,0],[0,1,0]])
+    print model.predict_proba([[1,0,0],[0,1,0],[0,0,1]])
     print model.feature_count_
 
-
+    print(metrics.classification_report(y, predicted))
+    print(metrics.confusion_matrix(y, predicted))
 
 def str2datetime(yyyymmdd):
     #print '%d%d%d'% (int(yyyymmdd[6:8]), int(yyyymmdd[4:6])-1 , int(yyyymmdd[0:4])) 
@@ -296,8 +303,8 @@ def set_biz_calendar():
     
     #print np.array(s1)
 if __name__ == '__main__':
-    #weather_play()
-    #test()
+    weather_play()
+    test()
     
     #
     # 
@@ -307,9 +314,9 @@ if __name__ == '__main__':
     # the dow jones or some other indices closed high on 
     # the previous trading day?
     #
-    rs = set_biz_calendar()
-    print ''.join('%s,\n' % rs[i] for i in range(5)), rs.after(str2datetime('20160324')),\
-                                                           datetime2ystr(rs.after(str2datetime('20160324')))
+    #rs = set_biz_calendar()
+    #print ''.join('%s,\n' % rs[i] for i in range(5)), rs.after(str2datetime('20160324')),\
+    #                                                       datetime2ystr(rs.after(str2datetime('20160324')))
      
     #ewh_hsi(rs)
-    predict(rs)
+    #predict(rs)

src/rethink/analytics_engine.py

@@ -34,13 +34,13 @@ class AnalyticsEngine(AbstractGatewayListener):
         
     
     def test_oc(self, oc2):
-        expiry = '20170529'
+        expiry = '20170830'
         contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', expiry, 0, '')
         contract = ContractHelper.makeContract(contractTuple)  
         
         oc2.set_option_structure(contract, 200, 50, 0.0012, 0.0328, expiry)        
         
-        oc2.build_chain(25280, 0.04, 0.22)
+        oc2.build_chain(27000, 0.04, 0.22)
         
 #         expiry='20170324'
 #         contractTuple = ('QQQ', 'STK', 'SMART', 'USD', '', 0, '')
@@ -59,13 +59,13 @@ class AnalyticsEngine(AbstractGatewayListener):
         
     
     def test_oc3(self, oc3):
-        expiry = '20170629'
+        expiry = '20170928'
         contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', expiry, 0, '')
         contract = ContractHelper.makeContract(contractTuple)  
          
         oc3.set_option_structure(contract, 200, 50, 0.0012, 0.0328, expiry)        
          
-        oc3.build_chain(25280, 0.04, 0.22)
+        oc3.build_chain(27000, 0.04, 0.22)
 
 #         expiry = '20170331'
 #         contractTuple = ('QQQ', 'STK', 'SMART', 'USD', '', 0, '')
@@ -92,7 +92,21 @@ class AnalyticsEngine(AbstractGatewayListener):
             self.tds.add_symbol(o)
         self.tds.add_symbol(oc3.get_underlying())
         
-    
+    def init_oc_from_params(self, **args):
+        oc = OptionsChain(args['name'])
+        contractTuple = (args['m_symbol'], args['m_secType'], args['m_exchange'], args['m_currency'], 
+                         args['m_expiry'], args['m_strike'], args['m_right'])
+        contract = ContractHelper.makeContract(contractTuple)  
+        #underlying, spd_size, multiplier, rate, div, expiry, trade_vol=0.15)
+        oc.set_option_structure(contract, args['spd_size'], args['multiplier'], args['rate'], args['div'], args['m_expiry'], 
+                                args['trade_vol'])                                                  
+        oc.build_chain(args['undly_price'], args['bound'], args['trade_vol'])
+        
+            
+        for o in oc.get_option_chain():
+            self.tds.add_symbol(o)
+        self.tds.add_symbol(oc.get_underlying())
+        return oc
     
     def start_engine(self):
         self.twsc.start_manager()
@@ -258,7 +272,10 @@ if __name__ == '__main__':
       'clear_offsets':  False,
       'logconfig': {'level': logging.INFO, 'filemode': 'w', 'filename': '/tmp/ae.log'},
       'topics': ['tickPrice', 'tickSize'],
-      'seek_to_end': ['*']
+      'seek_to_end': ['*'],
+#       'ocs':[
+#             name, underlying, spd_size, multiplier, rate, div, expiry, trade_vol)
+#             ]
 
       #'seek_to_end':['tickSize', 'tickPrice','gw_subscriptions', 'gw_subscription_changed']
       }
@@ -295,4 +312,4 @@ if __name__ == '__main__':
     server.start_engine()
     
           
-        
+        

src/rethink/option_chain_chart.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+import sys, traceback, logging, json
+from rethink.portfolio_item import PortfolioRules, PortfolioItem, Portfolio
+import numpy as np
+from rethink.table_model import AbstractTableModelListener, AbstractTableModel
+from rethink.option_chain import OptionsChain
+    
+    
+class OptionChainChartTM(OptionsChain, AbstractTableModel, AbstractTableModelListener):
+    
+    
+    def __init__(self, name, pf,  kproducer):
+        
+        OptionsChain.__init__(self, pf)
+        AbstractTableModel.__init__(self)
+        AbstractTableModelListener.__init__(self, name)
+        self.request_ids = {}
+        self.kproducer = kproducer
+        
+        
+    def get_kproducer(self):
+        return self.kproducer
+      
+#     def get_column_count(self):
+#         raise NotImplementedException
+#     
+#     def get_row_count(self):
+#         raise NotImplementedException
+# 
+#     def get_column_name(self, col):
+#         raise NotImplementedException
+# 
+#     def get_column_id(self, col):
+#         raise NotImplementedException
+# 
+#     def get_value_at(self, row, col):
+#         raise NotImplementedException
+#     
+#     def get_values_at(self, row):
+#         raise NotImplementedException
+#     
+#     def set_value_at(self, row, col, value):
+#         raise NotImplementedException
+#     
+#     def insert_row(self, values):
+#         raise NotImplementedException
+
+
+    def event_tm_table_structure_changed(self, event, source, origin_request_id, account, data_table_json):
+        logging.info("[OptionChainChartTM:%s] received %s content:[%s]" % (self.name, event, vars()))
+        
+        
+    
+    def event_tm_request_table_structure(self, event, request_id, target_resource, account):
+        self.request_ids[request_id] = {'request_id': request_id, 'account': account}
+        logging.info("[OptionChainChartTM:%s] received %s content:[%s]" % (self.name, event, vars()))
+        
+        logging.info("[OptionChainChartTM: self class: %s, incoming target class %s" % (self.get_object_name()['class'], target_resource['class']))
+        if target_resource['class'] == self.get_object_name()['class']:
+            self.get_kproducer().send_message(AbstractTableModel.EVENT_TM_TABLE_STRUCTURE_CHANGED,                                               
+                                      json.dumps({'source': self.get_object_name(), 
+                                                  'origin_request_id': request_id, 'account': account, 
+                                                  'data_table_json': self.get_JSON()}))
+                
+               
+    

src/rethink/table_model.py

@@ -61,11 +61,7 @@ class AbstractTableModel(Publisher):
         raise NotImplementedException
 
 
-class AbstractPortfolioTableModelListener(BaseMessageListener):
-    '''
-    
-    '''
-    
+class AbstractTableModelListener(BaseMessageListener):
     def __init__(self, name):
         BaseMessageListener.__init__(self, name)
         
@@ -85,6 +81,13 @@ class AbstractPortfolioTableModelListener(BaseMessageListener):
         
     def event_tm_request_table_structure(self, event, request_id, target_resource, account):
         logging.info("[%s] received %s content:[%s]" % (self.name, event, vars()))        
+
+class AbstractPortfolioTableModelListener(AbstractTableModelListener):
+    pass
+
+
+
+
     
 
 

src/sh/ae.sh

@@ -11,5 +11,5 @@ else
 	FINOPT_HOME=~/l1304/workspace/finopt-ironfly/finopt/src
 fi
 export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
-#python $FINOPT_HOME/rethink/analytics_engine.py  -c -g AE1  
-python $FINOPT_HOME/rethink/analytics_engine.py   -g AE1  
+python $FINOPT_HOME/rethink/analytics_engine.py  -c -g AE1  
+#python $FINOPT_HOME/rethink/analytics_engine.py   -g AE1  

src/sh/start_twsgw.sh

@@ -18,11 +18,11 @@ export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
 
 #
 # clear offsets in redis / reload saved subscription entries
-#python $FINOPT_HOME/comms/ibgw/tws_gateway.py  -c -f $FINOPT_HOME/config/tws_gateway.cfg 
+python $FINOPT_HOME/comms/ibgw/tws_gateway.py  -c -f $FINOPT_HOME/config/tws_gateway.cfg 
 
 
 # restart gateway keep the redis offsets but erase the subscription entries
-python $FINOPT_HOME/comms/ibgw/tws_gateway.py  -r -f $FINOPT_HOME/config/tws_gateway.cfg 
+#python $FINOPT_HOME/comms/ibgw/tws_gateway.py  -r -f $FINOPT_HOME/config/tws_gateway.cfg 
 
 # normal restart - keep the offsets and reload from saved subscription entries
 #python $FINOPT_HOME/comms/ibgw/tws_gateway.py   -f $FINOPT_HOME/config/tws_gateway.cfg