updated option month handling, new columns in web page

alerts/fund_alerts.py

@@ -143,13 +143,14 @@ def retrieve_hk_holidays(year):
         tds = soup.findAll('h3')[0].parent.findAll('td', 'date')
         
         d1 = map(lambda x: (x.text.split(' ')[0], x.text.split(' ')[1]), tds[1:])
-        return map(lambda x: strftime('%Y%m%d', time.strptime('%s %s %s' % (month_names.index(x[1])+1, x[0], '2015'), "%m %d %Y")), d1)
+        
+        return map(lambda x: strftime('%Y%m%d', time.strptime('%s %s %s' % (month_names.index(x[1])+1, x[0], year), "%m %d %Y")), d1)
     except:
         print 'error'
 
 if __name__ == '__main__':      
     #send_daily_alert()
-    print retrieve_hk_holidays(2015)
+    print retrieve_hk_holidays(2016)
      
 #     print allianz()
 #     

config/app-20160611.cfg.bak

@@ -0,0 +1,135 @@
+[global]
+server.socket_host: "0.0.0.0"
+server.socket_port: 8082
+
+[redis]
+redis.server: "localhost"
+redis.port: 6379
+redis.db: 3
+redis.sleep: 0.5
+redis.datastore.key.option_implv: 'opt_implv'
+redis.datastore.key.option_chains: 'opt_chains'
+redis.datastore.key.option_set: 'opt_set'
+redis.datastore.key.option_implv_ts_set: 'opt_implv_ts_set'
+redis.datastore.key.option_implv_ts: 'opt_implv_ts' 
+
+# a redis key that stores all the contract ids of contracts in a portfolio  
+redis.datastore.key.port_conid_set: 'port_conid_set' 
+redis.datastore.key.port_prefix: 'PT'
+# note that the port summary key has no prefix assigned to it
+# to retrive its value just use rs.get instead of self.rs_<xxx>
+redis.datastore.key.port_summary: 'port_summary'
+redis.datastore.key.port_items: 'port_items'
+redis.datastore.key.acct_summary: 'acct_summary'
+ 
+[/]
+tools.sessions.on : True
+tools.staticdir.root : '/home/larry-13.04/workspace/finopt/src/'
+#tools.staticdir.root : '/home/larry-13.04/production/finopt/'
+
+[/static]
+tools.staticdir.on : True
+tools.staticdir.tmpl : './html'
+#tools.staticdir : './html'
+
+[/public]
+tools.staticdir.on: True
+tools.staticdir.dir : './html/public'
+
+[/ws]
+tools.websocket.on: True
+tools.websocket.handler_cls: opt_serve.OptWebSocket
+
+
+
+[options_data]
+options_data.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/opt.log', 'filemode': 'w','level': logging.DEBUG}"
+
+[portfolio]
+portfolio.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/port.log', 'filemode': 'w','level': logging.INFO}"
+portfolio.epc: "{'stream_to_Kafka': True}"   
+portfolio.account_summary_tags: "['AccountType','NetLiquidation','TotalCashValue','SettledCash','AccruedCash','BuyingPower','EquityWithLoanValue','PreviousDayEquityWithLoanValue','GrossPositionValue','RegTEquity','RegTMargin','SMA','InitMarginReq','MaintMarginReq','AvailableFunds','ExcessLiquidity','Cushion','FullInitMarginReq','FullMaintMarginReq','FullAvailableFunds','FullExcessLiquidity','LookAheadNextChange','LookAheadInitMarginReq','LookAheadMaintMarginReq','LookAheadAvailableFunds','LookAheadExcessLiquidity','HighestSeverity','DayTradesRemaining','Leverage']"
+
+
+[opt_serve]
+opt_serve.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/serve.log', 'filemode': 'w','level': logging.INFO}"
+
+
+[cep]
+kafka.host: 'vsu-01'
+kafka.port: 9092
+kafka.ib.topic.tick_price: 'ib_tick_price'
+kafka.ib.topic.tick_size: 'ib_tick_size'
+ib.subscription.fileloc: '/home/larry-13.04/workspace/finopt/data/subscription.txt'
+
+
+[market]
+hkex.openhours: '{"morning":[915,1200], "afternoon":[1300,1615]}'
+#ib.gateway: '127.0.0.1'
+ib.gateway: 'vsu-01'
+ib.port: 7496
+#gw port
+#ib.port:4001
+ib.appid.portfolio: 9922
+
+ib.appid: 9911
+option.underlying = "('HSI', 'FUT', 'HKFE', 'HKD', '', 0, '')"
+option.underlying.month_price = "[['20151029', 22817.0, '^HSI'], ['20151127', 22715.0, '^HSI']]"
+option.underlying.yahoo_ws = "{'use_yahoo': True, 'func': 'ystockquote.get_price'}"
+option.underlying.tick_size = 200
+
+# refer to this link for rate and div
+# https://www.hkex.com.hk/eng/sorc/tools/calculator_index_option.aspx
+option.greeks.recal = "{'use_last_if_no_bidask': True, 'rate':0.0009, 'div':0.328, 'vol':0.2}"
+option.chain_range = 0.08
+option.bid_ask_spread_tolerance = 0.90
+
+[alert_bot]
+msg_bot.jid: "robo@route69.hopto.org"
+msg_bot.pass: 123
+msg_bot.recipients: "['blueman@route69.hopto.org']"
+msg_bot.redis_mq: 'chatq'
+msg_bot.redis_prefix: 'alert_bot'  
+#
+# 'filename': '../log/alert_bot.log', 'filemode': 'w', 
+msg_bot.logconfig: "{'level': logging.INFO}"
+
+[epc]
+kafka.host: 'vsu-01'
+kafka.port: 9092
+
+[ib_mds]
+ib_mds.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/ib_mds.log', 'filemode': 'w','level': logging.INFO}"
+ib_mds.ib_port: 7496
+#ib_mds.ib_port: 4001
+ib_mds.appid.id: 9800
+ib_mds.gateway: 'localhost'
+#ib_mds.gateway: '192.168.1.118'
+ib_mds.is_persist: 1
+ib_mds.persist_dir: '/home/larry-13.04/workspace/finopt/data/mds_files'
+ib_mds.spill_over_limit: 10000
+
+[ib_heartbeat]
+ib_heartbeat.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/ib_mds.log', 'filemode': 'w','level': logging.INFO}"
+#ib_heartbeat.ib_port: 4001
+ib_heartbeat.ib_port: 7496
+ib_heartbeat.appid.id: 9911
+#ib_heartbeat.gateway: 'localhost'
+ib_heartbeat.gateway: '192.168.1.118'
+ib_heartbeat.try_interval: 60
+ib_heartbeat.suppress_msg_interval: 120
+
+[smart_order]
+smart_order.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/smart_order.log', 'filemode': 'w','level': logging.INFO}"
+
+
+[tws_gateway]
+subscription_manager.subscriptions.redis_key: 'subscriptions'  
+tws_gateway.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/tws_gateway.log', 'filemode': 'w','level': logging.INFO}"
+tws_gateway.order_transmit: False
+
+
+[options_chain]
+options_calculation_engine.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/oce.log', 'filemode': 'w','level': logging.INFO}"
+option_chain_id.redis_key_prefix: 'optchain-'
+clear_redis_on_start: True

config/app.cfg

@@ -1,11 +1,15 @@
 [global]
 server.socket_host: "0.0.0.0"
-server.socket_port: 8082
+#server.socket_port: 8082
+server.socket_port: 8091
 
 [redis]
 redis.server: "localhost"
 redis.port: 6379
-redis.db: 3
+# db - 0 production larry046, 3 development, 2 production mchan927
+#redis.db: 3
+redis.db:2
+
 redis.sleep: 0.5
 redis.datastore.key.option_implv: 'opt_implv'
 redis.datastore.key.option_chains: 'opt_chains'
@@ -21,6 +25,8 @@ redis.datastore.key.port_prefix: 'PT'
 redis.datastore.key.port_summary: 'port_summary'
 redis.datastore.key.port_items: 'port_items'
 redis.datastore.key.acct_summary: 'acct_summary'
+redis.datastore.key.hkex_holiday_prefix: 'hkex_holiday_'
+
  
 [/]
 tools.sessions.on : True
@@ -65,16 +71,29 @@ ib.subscription.fileloc: '/home/larry-13.04/workspace/finopt/data/subscription.t
 
 [market]
 hkex.openhours: '{"morning":[915,1200], "afternoon":[1300,1615]}'
-#ib.gateway: '127.0.0.1'
-ib.gateway: 'vsu-01'
-ib.port: 7496
+
+
+
+ib.gateway: '127.0.0.1'
+#ib.gateway: 'vsu-01'
+
+# 7496 - production larry046, 7496 - development,  8496 production mchan927
+ib.port: 8496
 #gw port
 #ib.port:4001
-ib.appid.portfolio: 9922
 
-ib.appid: 9911
+# 9922: production larry046, 9922 development, 9933 production mchan927
+ib.appid.portfolio: 9933
+
+# 9911: production larry046, 9911 development, 9913 production mchan927
+ib.appid: 9913
+
 option.underlying = "('HSI', 'FUT', 'HKFE', 'HKD', '', 0, '')"
-option.underlying.month_price = "[['20151029', 22817.0, '^HSI'], ['20151127', 22715.0, '^HSI']]"
+
+# the month value is no longer required as the system
+# will auto retrieve holidays from hk gov website
+# and deduce the near and next month options last trading date
+option.underlying.month_price = "[['20160629', 20100, '^HSI'], ['20160728', 20100.0, '^HSI']]"
 option.underlying.yahoo_ws = "{'use_yahoo': True, 'func': 'ystockquote.get_price'}"
 option.underlying.tick_size = 200
 

finopt/optcal.py

@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 
 from QuantLib import *
-
+from bs4 import BeautifulSoup
+from urllib2 import urlopen, Request
+from time import strftime
+import time
+import traceback
 
 
 def cal_implvol(spot, strike, callput, evaldate, exdate, rate, div, vol, premium):
@@ -71,11 +75,132 @@ def str2qdate(yyyymmdd):
     return Date(int(yyyymmdd[6:8]), months[int(yyyymmdd[4:6])-1 ], int(yyyymmdd[0:4]))
     
 
+def qdate2str(dd):
+    return '%s%s%s' % (dd.ISO()[0:4], dd.ISO()[5:7], dd.ISO()[8:10])
+    
+
 def str2qopt_type(callput):
     if callput.upper() == 'C':
         return Option.Call
     return Option.Put
 
+
+
+def get_hk_holidays(year):
+    month_names = {'January' : 1,
+                    'February': 2,
+                    'March': 3,
+                    'April': 4,
+                    'May': 5,
+                    'June': 6,
+                    'July': 7,
+                    'August': 8,
+                    'September': 9,
+                    'October': 10,
+                    'November': 11,
+                    'December': 12,
+                    } 
+    try:
+        url = 'http://www.gov.hk/en/about/abouthk/holiday/{{year}}.htm'
+        url = url.replace('{{year}}', str(year))
+
+
+        headers = { 'User-Agent' : 'Mozilla/5.0' }
+        req = Request(url, None, headers)
+        html = urlopen(req).read()
+
+        
+                
+        
+        
+        soup = BeautifulSoup(html, 'html5lib')
+        
+        tds = soup.findAll('h3')[0].parent.findAll('td', 'date')
+        
+        d1 = map(lambda x: (int(x.text.split(' ')[0]), x.text.split(' ')[1]), tds[1:])
+        holidays =  map(lambda x: '%d%02d%02d' % (year, int(month_names[x[1]]), int(x[0]) ), d1)
+        #return map(lambda x: strftime('%Y%m%d', time.strptime('%s %s %s' % (month_names.index(x[1])+1, x[0], year), "%m %d %Y")), d1)
+        #print d1
+        print holidays
+        
+        return holidays
+        
+    except:
+        traceback.print_exc()
+    
+
+
+
+def get_HSI_last_trading_day(holidays, month, year):
+
+    cal = HongKong()
+    map(lambda x: cal.addHoliday(str2qdate(x)), holidays)    
+    
+    def deduce_last_trading_day(ld):
+        
+        ld = ld - 1
+        #print '###' + str(ld)
+        if cal.isHoliday(ld):
+            return deduce_last_trading_day(ld - 1)
+        elif not cal.isBusinessDay(ld):
+            return deduce_last_trading_day(ld - 1)
+        else:
+            #print '---' + str(ld)
+            return ld
+    
+    return qdate2str(deduce_last_trading_day(Date.endOfMonth(Date(1, month, year))))
+    
+    
+# QUANTLIB Period class usage:
+# https://ipythonquant.wordpress.com/2015/04/04/a-brief-introduction-to-the-quantlib-in-python/
+# check usage below:
+    
+    
+# def get_HSI_expiry(year):
+# 
+#     month_names = [January,
+#                     February,
+#                     March,
+#                     April,
+#                     May,
+#                     June,
+#                     July,
+#                     August,
+#                     September,
+#                     October,
+#                     November,
+#                     December,
+#                     ]    
+#     mm_dd = {}
+#     # load calendar
+#     holidays = ['20160101', '20160208', '20160209',  '20160210', '20160325', '20160326', '20160328', '20160404', '20160502', '20160514', '20160609', '20160701', '20160916', '20161001', '20161010', '20161226', '20161227']
+#     chk = HongKong()
+#     
+#     
+#     map(lambda x: chk.addHoliday(Date(int(x[6:8]), month_names[int(x[4:6])-1], int(x[0:4]))), holidays)
+#     
+#     
+#     #print Date.todaysDate() - 1
+#     #chk.addHoliday(Date(24, December, 2015))
+#     def deduce_last_trading_day(ld):
+#         # ld -=1
+#         ld = ld - 1
+#         
+#         if chk.isHoliday(ld):
+#             return deduce_last_trading_day(ld - 1)
+#         elif not chk.isBusinessDay(ld):
+#             return deduce_last_trading_day(ld - 1)
+#         
+#         print ld
+#         return ld
+#         
+#     
+#         
+#     return map(lambda x: deduce_last_trading_day(Date.endOfMonth(Date(1, x, year) + Period("1m"))), range(1, 12))
+#     #mm_dd = deduce_last_trading_day(Date.endOfMonth(Date.todaysDate())) 
+#     #return mm_dd
+    
+
 if __name__ == '__main__':
     
     
@@ -145,15 +270,38 @@ if __name__ == '__main__':
         
     #print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
 
+# 
+#     chk = HongKong()
+#     chk.addHoliday(Date(24, December, 2015))
+#     chk.addHoliday(Date(19, October, 2015))
+#     chk.addHoliday(str2qdate('20151020'))
+#     print chk.isBusinessDay(Date(24, December, 2015))
+#     print chk.isBusinessDay(Date(19, October, 2015))
+#     print chk.isEndOfMonth(Date(29, October, 2015))
+#     print chk.isEndOfMonth(Date(30, October, 2015))
+#     print chk.advance(Date(17, October, 2015), 1, 0)
+#     print chk.advance(Date(17, October, 2015), 1, 1)
+#     print chk.advance(Date(17, October, 2015), 1, 2)
+    #print get_HSI_expiry(2016)
+    
+    holidays = get_hk_holidays(2016)
 
-    chk = HongKong()
-    chk.addHoliday(Date(24, December, 2015))
-    chk.addHoliday(Date(19, October, 2015))
-    chk.addHoliday(str2qdate('20151020'))
-    print chk.isBusinessDay(Date(24, December, 2015))
-    print chk.isBusinessDay(Date(19, October, 2015))
-    print chk.isEndOfMonth(Date(29, October, 2015))
-    print chk.isEndOfMonth(Date(30, October, 2015))
-    print chk.advance(Date(17, October, 2015), 1, 0)
-    print chk.advance(Date(17, October, 2015), 1, 1)
-    print chk.advance(Date(17, October, 2015), 1, 2)
+    
+    
+    month_names = [January,
+                February,
+                March,
+                April,
+                May,
+                June,
+                July,
+                August,
+                September,
+                October,
+                November,
+                December,
+                ] 
+    for i in month_names:
+        dd = get_HSI_last_trading_day(holidays, i, 2016)
+        print dd
+        

finopt/options_data.py

@@ -63,7 +63,10 @@ class OptionsMarketDataManager():
         # option.chain_range = 0.08
         
         undly_tuple = eval(config.get("market", "option.underlying").strip('"').strip("'"))
-        undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+        #undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+        # 20160612
+        undly_months_prices = DataMap().rskeys['option.underlying.month_price']
+        
         undly_yahoo_ws = eval(config.get("market", "option.underlying.yahoo_ws").strip('"').strip("'"))
         self.cal_greeks_config = eval(config.get("market", "option.greeks.recal").strip('"').strip("'"))
     
@@ -486,6 +489,7 @@ class DataMap():
     rs = None
     rskeys = {}
     mkt_sessions = {}
+    
     config = None
 
 
@@ -511,6 +515,54 @@ class DataMap():
         
         DataMap.rs = redis.Redis(host, port, db)
         logging.info(self.rs.info())
+        
+        
+        self.rskeys['option.underlying.month_price'] = self.set_option_calendar()
+
+
+    # 20160612
+    # routine to determine the near and far month contract expiry date
+    #
+    # logic:
+    #      - determine today's date
+    #
+    #      - auto mode:
+    #      - look up entries in redis
+    #      - if records are found, use the records in redis
+    #         else 
+    #            retrive record from hkgov website
+    #            update redis db
+    #      -  
+    
+    def set_option_calendar(self):
+        year = int(datetime.datetime.now().strftime('%Y'))
+        month = int(datetime.datetime.now().strftime('%m'))
+
+
+        holiday_key_prefix = config.get("redis", "redis.datastore.key.hkex_holiday_prefix").strip('"').strip("'")        
+        
+        rs = self.redisConn()
+        holiday_key = '%s%s' % (holiday_key_prefix, year)
+        holidays = rs.get(holiday_key)
+                    
+        if holidays == None:
+            holidays = optcal.get_hk_holidays(year)
+            logging.info("options_data:set_option_calendar: retrieved from gov.hk --> update Redis: [%s]: %s", holiday_key, json.dumps(holidays))
+            rs.set(holiday_key, json.dumps(holidays))    
+
+        else:
+            holidays = json.loads(holidays)
+            logging.info("options_data:set_option_calendar: retrieved holidays from redis %s", str(holidays))
+
+        
+        
+        undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+        
+        undly_months_prices[0][0] = optcal.get_HSI_last_trading_day(holidays, month, year)
+        undly_months_prices[1][0] = optcal.get_HSI_last_trading_day(holidays, (month + 1) % 12, year)
+        
+        logging.info("options_data:set_option_calendar:  %s " % str(undly_months_prices))
+        return undly_months_prices
     
     def redisConn(self):
         return DataMap.rs
@@ -982,8 +1034,12 @@ def create_subscription_file(dest):
     f1.close()
 
 
+
 def console(config, omd):
-    undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+#    undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+# 20160612
+    undly_months_prices = DataMap().rskeys['option.underlying.month_price']
+    
     done = False
     while not done:
 #        try:
@@ -1041,7 +1097,9 @@ def add_portfolio_subscription(config, omd):
     p.retrieve_position()
     
     
-    undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+#    undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+    # 20160612
+    undly_months_prices = DataMap().rskeys['option.underlying.month_price']
     
     # example: MHI-20150929-22600-P
     toks = map(lambda x: x.split('-'), p.get_pos_contracts())

finopt/portfolio.py

@@ -336,7 +336,11 @@ class PortfolioManager():
         # new instruments to its subscription list 
 
         pall = set(self.r_conn.keys(pattern='%s*' % self.rs_port_keys['port_prefix']))
-        s = '["symbol","right","avgcost","spotpx","pos","delta","theta","pos_delta","pos_theta","unreal_pl","last_updated"],'
+        
+        # 2016/06/11
+        # add 2 new columns: avgcost in points, unreal_pl ratio
+        #s = '["symbol","right","avgcost","spotpx","pos","delta","theta","pos_delta","pos_theta","unreal_pl","last_updated"],'
+        s = '["symbol","right","avgcost","avgpx","spotpx","pos","delta","theta","pos_delta","pos_theta","unreal_pl","unreal%","last_updated"],'
         
         def split_toks(x):
             try: # 
@@ -344,9 +348,16 @@ class PortfolioManager():
                 #print pmap
                 gmap = json.loads(self.r_conn.get(x[3:]))
                 #print gmap
-                s = '["%s","%s",%f,%f,%f,%f,%f,%f,%f,%f,"%s"],' % (x[3:], x[len(x)-1:], pmap['6001'], gmap['5006'], pmap['6002'],\
+#                 s = '["%s","%s",%f,%f,%f,%f,%f,%f,%f,%f,"%s"],' % (x[3:], x[len(x)-1:], pmap['6001'], gmap['5006'], pmap['6002'],\
+#                                                                              gmap['5002'],gmap['5004'],\
+#                                                                              pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated'])
+        # 2016/06/11        
+                s = '["%s","%s",%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,"%s"],' % (x[3:], x[len(x)-1:], pmap['6001'], pmap['6001'] / pmap['6007'], gmap['5006'], pmap['6002'],\
                                                                              gmap['5002'],gmap['5004'],\
-                                                                             pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated'])
+                                                                             pmap['6005'],pmap['6006'],pmap['6008'],
+                                                                             
+                                                                             (gmap['5002'] - (pmap['6001'] / pmap['6007'])) / pmap['6001'] / pmap['6007'],
+                                                                             pmap['last_updated'])                
             except:
                 logging.error('entry %s skipped due to an exception. Please validate your position' % x)
                 return ''