tws gateway

cep/greeks_changes.py

@@ -0,0 +1,193 @@
+import sys
+
+from pyspark import SparkContext
+from pyspark.streaming import StreamingContext
+from pyspark.streaming.kafka import KafkaUtils
+from numpy import *
+import pylab
+from scipy import stats
+import time, datetime
+import threading
+import time
+import os
+from finopt import ystockquote
+##
+##
+##
+## This example demonstrates the use of accumulators and broadcast 
+## and how to terminate spark running jobs
+## 
+## it also demonstrates how to send alerts via xmpp
+## (requires prosody server running and redisQueue)
+##
+##
+
+##
+##
+## insert the path so spark-submit knows where
+## to look for a file located in a given directory
+##
+## the other method is to export PYTHONPATH before 
+## calling spark-submit
+##
+# import sys
+# sys.path.insert(0, '/home/larry-13.04/workspace/finopt/cep')
+print sys.path
+
+
+#import optcal
+import json
+import numpy
+#from finopt.cep.redisQueue import RedisQueue
+from comms.redisQueue import RedisQueue
+from comms.alert_bot import AlertHelper
+
+
+def f1(time, rdd):
+    lt =  rdd.collect()
+    if not lt:
+        return
+    print '**** f1'
+    print lt
+    print '**** end f1'
+    f = open('/home/larry/l1304/workspace/finopt/data/mds_files/std/std-20151008.txt', 'a') # % datetime.datetime.now().strftime('%Y%m%d%H%M'), 'a')
+    msg = ''.join('%s,%s,%s,%s,%s\n'%(s[0], s[1][0][0].strftime('%Y-%m-%d %H:%M:%S.%f'),s[1][0][1],s[1][0][2], s[1][1]) for s in lt)
+    f.write(msg)
+    d = Q.value
+    
+    # return rdd tuple (-,((-,-),-)): name = 0--, time 100, sd 101, mean 102, vol 11-
+    
+    for s in lt:
+        if s[0].find('HSI-20151029-0') > 0 and (s[1][0][1] > 4.5 or s[1][1] > 100000):      
+            msg  = 'Unusal trading activity: %s (SD=%0.2f, mean px=%d, vol=%d) at %s\n'\
+                 % (s[0], \
+                    s[1][0][1], s[1][0][2],\
+                    s[1][1],\
+                    s[1][0][0].strftime('%m-%d %H:%M:%S'))   
+            q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'], d['port'], d['db'])
+            q.put(msg)
+            
+
+def f2(time, rdd):
+    lt =  rdd.collect()
+    if lt:
+        change = lt[0][0]
+        d = Q.value
+        print '********** f2'
+        print lt[0][0], Threshold.value, lt[0][1]
+        print '********** end f2'
+
+        
+        if change > Threshold.value:
+            msg = 'Stock alert triggered: %0.6f, mean: %0.2f' % (change, lt[0][1])
+            print msg
+#             q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'], d['port'], d['db'])
+#             q.put(msg)
+    
+
+def f3(time, rdd):
+    lt =  rdd.collect()
+    if lt:
+        #print '%s %0.2f %0.2f' % (lt[0], lt[1][0], lt[1][1])
+        for s in lt:
+            print '%s [%s] ' % (s[0], ','.join('(%0.4f, %0.4f)' % (e[0], e[1]) for e in s[1]))
+    
+
+# to run from command prompt
+# 0. start kafka broker
+# 1. edit subscription.txt and prepare 2 stocks
+# 2. run ib_mds.py 
+# 3. spark-submit  --jars spark-streaming-kafka-assembly_2.10-1.4.1.jar ./alerts/pairs_corr.py vsu-01:2181 
+
+# http://stackoverflow.com/questions/3425439/why-does-corrcoef-return-a-matrix
+# 
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print("Usage: %s <broker_list ex: vsu-01:2181>  <rdd_name> <tick id> <fn name>" % sys.argv[0])
+        print("Usage: to gracefully shutdown type echo 1 > /tmp/flag at the terminal")
+        exit(-1)
+
+
+
+    app_name = "std_deviation_analysis"
+    sc = SparkContext(appName= app_name) #, pyFiles = ['./cep/redisQueue.py'])
+    ssc = StreamingContext(sc, 2)
+    ssc.checkpoint('/home/larry-13.04/workspace/finopt/log/checkpoint')
+
+
+
+    brokers, qname, id, fn  = sys.argv[1:]
+    id = int(id)
+    
+    #
+    # demonstrate how to use broadcast variable
+    #
+    NumProcessed = sc.accumulator(0)
+    
+    cls = float(ystockquote.get_historical_prices('^HSI', '20151005', '20151005')[1][4])
+    
+    print 'closing price of HSI %f' % cls
+    
+    Q = sc.broadcast({'cls': cls, \
+                      'rname': 'rname', 'qname': qname, 'namespace': 'mdq', 'host': 'localhost', 'port':6379, 'db': 3, 'alert_bot_q': ('alert_bot', 'chatq')})
+    Threshold = sc.broadcast(0.25)
+    #kvs = KafkaUtils.createDirectStream(ssc, ['ib_tick_price', 'ib_tick_size'], {"metadata.broker.list": brokers})
+    kvs = KafkaUtils.createStream(ssc, brokers, app_name, {'optionAnalytics':1})
+
+    lns = kvs.map(lambda x: x[1])
+    
+#{"analytics":{"imvol" : 0.210757782404, "vega" : 3321.50906944, "delta" : 0.402751602804, "theta" : -5.58857173887, "npv" : 499.993413708, "gamma" : 0.00021240629952}, "contract":{"m_conId": 0, "m_right": "C", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_multiplier": 50, "m_expiry": "20160128", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 22600.0}, "tick_values":{"0" : 20, "1" : 500.0, "2" : 510.0, "3" : 25, "4" : 500.0, "5" : 1, "8" : 22, "9" : 628.0}, "extra":{"spot" : 22190.0, "rate" : 0.0012, "last_updated" : "20151204143050", "div" : 0.0328}}    
+
+    mdp = lns.map(lambda x: json.loads(x))\
+            .filter(lambda x: (x['extra']['chain_id'] == 'HSI-DEC30'))\
+            .map(lambda x: (x['contract']['m_strike'], (x['analytics']['imvol'], x['analytics']['theta'])  ))\
+            .groupByKeyAndWindow(6, 4, 1)
+            #.groupByKeyAndWindow(12, 10, 1)
+
+#     mds = lns.map(lambda x: json.loads(x))\
+#             .filter(lambda x: (x['typeName'] == 'tickSize'))\
+#             .map(lambda x: (x['contract'], x['size'] ))\
+#             .reduceByKeyAndWindow(lambda x, y: (x + y), None, 12, 10, 1)
+#     s1 = mdp.map(lambda x: (x[0], (datetime.datetime.fromtimestamp( [a[0] for a in x[1]][0]  ), numpy.std([a[1] for a in x[1]]),\
+#                  numpy.mean([a[1] for a in x[1]]))\
+#                  )) 
+    
+#     mds.pprint()            
+#     sps = s1.join(mds)
+#     sps.foreachRDD(f1)
+
+#    mdp.pprint()
+    mdp.foreachRDD(f3)
+    #trades.foreachRDD(eval(fn))
+    
+        
+    def do_work():
+
+        while 1:
+            # program will stop after processing 40 rdds
+#             if NumProcessed.value == 70:
+#                 break            
+            # program will stop on detecting a 1 in the flag file
+            try:
+                f = open('/tmp/flag')
+                l = f.readlines()
+                print 'reading %s' % l[0]
+                if '1' in l[0]:
+                    os.remove('/tmp/flag') 
+                    print 'terminating..........'        
+                    ssc.stop(True, False) 
+                    sys.exit(0)          
+                f.close()
+                time.sleep(2)
+            except IOError:
+                continue
+            
+            
+        
+    t = threading.Thread(target = do_work, args=())
+    t.start()
+    ssc.start()
+    ssc.awaitTermination()
+    
+

comms/epc.py

@@ -119,7 +119,7 @@ class ExternalProcessComm(threading.Thread):
                                          message.offset, message.key,
                                          message.value))
 
-            print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+            print ("received %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
                                          message.offset, message.key,
                                          message.value))
 
@@ -141,4 +141,9 @@ if __name__ == '__main__':
     e = ExternalProcessComm(config)
     e.start()
     
-    e.post_msg(ExternalProcessComm.EPC_TOPICS['EPC_PORT_SUMMARY_TOPIC'], 'test msg')
+    e.post_msg(ExternalProcessComm.EPC_TOPICS['EPC_PORT_SUMMARY_TOPIC'], 'test msg')
+    
+    e2 = ExternalProcessComm(config)
+    e2.start()
+
+    

comms/ib_heartbeat.py

@@ -82,6 +82,8 @@ class IbHeartBeat():
 
 
 
+        
+    
 
 if __name__ == '__main__':
            

comms/sample_tws_client.py

@@ -0,0 +1,580 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import thread, threading
+from threading import Lock
+import ConfigParser
+
+
+from time import sleep
+import time, datetime
+from ib.ext.Contract import Contract
+from ib.ext.Order import Order
+from ib.ext.ExecutionFilter import ExecutionFilter
+from random import randint
+
+from finopt.options_analytics import AnalyticsListener
+
+
+from misc2.helpers import ContractHelper, OrderHelper, ExecutionFilterHelper
+from finopt.options_chain import OptionsChain
+from comms.tws_client import SimpleTWSClient
+
+
+from kafka import KafkaConsumer
+from comms.tws_protocol_helper import TWS_Protocol
+
+
+
+class SampleClient(SimpleTWSClient):
+
+    tickerMap = {}
+    
+
+    def dump(self, msg_name, mapping):
+        # the mapping is a comms.tws_protocol_helper.Message object
+        # which can be accessed directly using the __dict__.['xxx'] method 
+        items = list(mapping.items())
+        items.sort()
+        print ('>>> %s <<< %s' % (msg_name, ''.join('%s=%s, '% (k, v if k <> 'ts' else datetime.datetime.fromtimestamp(v).strftime('%Y-%m-%d %H:%M:%S.%f')) for k, v in items))) 
+    
+    def accountSummaryEnd(self, items):
+        self.dump('accountSummaryEnd', items)
+        
+    def accountSummary(self, items):
+        self.dump('accountSummary', items)
+    # override the tickSize message
+    def tickSize(self, items):
+        try:
+            contract = self.tickerMap[items.__dict__['tickerId']]
+            field = items.__dict__['field']
+            ct =   ContractHelper.kv2object(contract, Contract)
+            print 'tickSize>> %s' % ('[%d:%s] %s=%0.4f [%s]' % \
+                                        (items.__dict__['tickerId'], ContractHelper.makeRedisKeyEx(ct),\
+                                        'bid' if field == 0 else ('ask' if field == 3 else ('last' if field == 5 else field)), \
+                                        items.__dict__['size'], datetime.datetime.fromtimestamp(items.__dict__['ts']).strftime('%Y-%m-%d %H:%M:%S.%f')))
+        except KeyError:
+            print 'tickSize: keyerror: (this could happen on the 1st run as the subscription manager sub list is still empty.'
+            print items
+
+
+    def tickPrice(self, items):
+        try:
+            contract = self.tickerMap[items.__dict__['tickerId']]
+            field = items.__dict__['field']
+            ct =   ContractHelper.kv2object(contract, Contract)
+            print 'tickPrice>> %s' % ('[%d:%s] %s=%0.4f [%s]' % \
+                                        (items.__dict__['tickerId'], ContractHelper.makeRedisKeyEx(ct),\
+                                        'bid_q' if field == 1 else ('ask_q' if field == 2 else ('last_q' if field == 4 else field)), \
+                                        items.__dict__['price'], datetime.datetime.fromtimestamp(items.__dict__['ts']).strftime('%Y-%m-%d %H:%M:%S.%f')))
+        except KeyError:
+            print 'tickPrice: keyerror:'
+            print items
+            
+            
+    def tickString(self, items):
+        pass
+
+    def tickGeneric(self, items):
+        pass
+
+    
+    def positionEnd(self, items):
+        self.dump('positionEnd', items)
+       
+
+    def position(self, items):
+        self.dump('position', items)
+        #pass
+
+    def error(self, items):
+        self.dump('error', items)
+
+    def error_0(self, items):
+        self.dump('error', items)
+ 
+    def error_1(self, items):
+        self.dump('error', items)
+        
+    def gw_subscriptions(self, items):
+        # <class 'comms.tws_protocol_helper.Message'>
+        # sample
+        #[[0, u'{"m_conId": 0, "m_right": "", "m_symbol": "HSI", "m_secType": "FUT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 0}'], [1, u'{"m_conId": 0, "m_right": "P", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 22200}'], [2, u'{"m_conId": 0, "m_right": "P", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 22400}'], [3, u'{"m_conId": 0, "m_right": "P", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 22600}'], [4, u'{"m_conId": 0, "m_right": "P", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 22800}'], [5, u'{"m_conId": 0, "m_right": "P", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 23000}'], [6, u'{"m_conId": 0, "m_right": "P", "m_symbol": "HSI", "m_secType": "OPT", "m_includeExpired": false, "m_expiry": "20151127", "m_currency": "HKD", "m_exchange": "HKFE", "m_strike": 23200}']]
+        #print items.__dict__['subscriptions']
+        
+        #l = map(lambda x: {x[0]: x[1]}, map(lambda x: (x[0], ContractHelper.kvstring2object(x[1], Contract)), items.__dict__['subscriptions']))
+        l = map(lambda x: {x[0]: x[1]}, map(lambda x: (x[0], json.loads(x[1])), items.__dict__['subscriptions']))
+        for i in l:
+            self.tickerMap.update(i)   
+        print 'gw_subscriptions -> dump tickerMap '
+        print self.tickerMap 
+    
+# override this function to perform your own processing
+#    def accountDownloadEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def execDetailsEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def updateAccountTime(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def deltaNeutralValidation(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def orderStatus(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def updateAccountValue(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def historicalData(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def openOrderEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def updatePortfolio(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def managedAccounts(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def contractDetailsEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def positionEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def bondContractDetails(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def accountSummary(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def updateNewsBulletin(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def scannerParameters(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickString(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def accountSummaryEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def scannerDataEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def commissionReport(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def error(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickGeneric(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickPrice(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def nextValidId(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def openOrder(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def realtimeBar(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def contractDetails(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def execDetails(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickOptionComputation(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def updateMktDepth(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def scannerData(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def currentTime(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def error_0(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def error_1(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickSnapshotEnd(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickSize(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def receiveFA(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def connectionClosed(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def position(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def updateMktDepthL2(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def fundamentalData(self, items):
+#        pass
+
+# override this function to perform your own processing
+#    def tickEFP(self, items):
+#        pass
+
+
+
+
+
+def on_ib_message(msg):
+    print msg
+
+
+
+
+
+
+def makeOrder(action, orderID, tif, orderType, price, qty):
+    newOptOrder = Order()
+    newOptOrder.m_orderId = orderID
+    newOptOrder.m_clientId = 0
+    newOptOrder.m_permid = 0
+    newOptOrder.m_action = action
+    newOptOrder.m_lmtPrice = price
+    newOptOrder.m_auxPrice = 0
+    newOptOrder.m_tif = tif
+    newOptOrder.m_transmit = True
+    newOptOrder.m_orderType = orderType
+    newOptOrder.m_totalQuantity = qty
+    return newOptOrder
+
+
+
+
+
+
+
+def test1():
+    twsc = SimpleTWSClient(host, port, '888')
+    twsc.registerAll([on_ib_message])
+    #twsc.get_command_handler().reqAccountSummary(100, 'All', "AccountType,NetLiquidation,TotalCashValue,BuyingPower,EquityWithLoanValue")
+     
+     
+    contract = Contract() #
+    contract.m_symbol = 'EUR'
+    contract.m_currency = 'USD'
+    contract.m_secType = 'CASH'
+    contract.m_exchange = 'IDEALPRO'
+    twsc.get_command_handler().reqMktData(contract)
+      
+    twsc.connect()
+    sleep(4)
+    twsc.disconnect()
+    print 'completed...'
+
+
+    
+def test2():
+    
+    contract = Contract() #
+    contract.m_symbol = 'EUR'
+    contract.m_currency = 'USD'
+    contract.m_secType = 'CASH'
+    contract.m_exchange = 'IDEALPRO'
+    
+   
+       
+    c = SampleClient(host, port, 'SampleClient-777')
+    c.connect()
+    c.get_command_handler().gw_req_subscriptions()
+    #c.get_command_handler().reqIds()
+    c.get_command_handler().reqMktData(contract)
+    
+    for i in range(567,568):
+        orderID = i
+ 
+        order = makeOrder( 'SELL', i, 'DAY', 'LMT', 1.0, randint(10,20) * 1000)
+        c.get_command_handler().placeOrder(orderID, contract, order)    
+    
+    
+    sleep(3)
+    c.get_command_handler().reqOpenOrders()
+    c.get_command_handler().reqExecutions()
+    c.get_command_handler().reqPositions()
+    
+    sleep(8)
+    c.disconnect()
+    print 'completed...'
+
+
+def test3():
+
+
+    c = SampleClient(host, port, 'SampleClient-777')
+    c.connect()
+    
+    
+    
+#     m_clientId = 0  # zero means no filtering on this field
+#     m_acctCode = ""
+#     m_time = ""
+#     m_symbol = ""
+#     m_secType = ""
+#     m_exchange = ""
+#     m_side = ""    
+    filter = ExecutionFilterHelper.kv2object({'m_time': '20151104  09:35:00'}, ExecutionFilter) 
+    c.get_command_handler().reqExecutions(filter)
+    sleep(7)    
+    
+#"yyyymmdd-hh:mm:ss"    
+    c.disconnect()
+
+
+
+def test4():
+    #global host, port
+    
+    f = open('/home/larry/l1304/workspace/finopt/data/subscription-nov-HSI.txt')
+    lns = f.readlines()
+    cs = map (lambda l: ContractHelper.makeContract(tuple(l.strip('\n').split(','))), lns)
+    c = SampleClient(host, port, 'SampleClient-777')
+    c.connect()
+    #for contract in cs:
+    #c.get_command_handler().reqMktData(cs[0])    
+#         
+#     contract = Contract() #
+#     contract.m_symbol = 'EUR'
+#     contract.m_currency = 'USD'
+#     contract.m_secType = 'CASH'
+#     contract.m_exchange = 'IDEALPRO'
+#     c.get_command_handler().reqMktData(contract)
+#     
+#     contract.m_symbol = 'HSI'
+#     contract.m_currency = 'HKD'
+#     contract.m_secType = 'OPT'
+#     contract.m_exchange = 'HKFE'
+#     contract.m_strike = 21000
+#     contract.m_multiplier = 50.0
+#     contract.m_includeExpired = False
+#     
+#     contract.m_right = 'P'
+#     contract.m_expiry = '20151127'
+    contract = Contract()
+    contract.m_symbol = 'GOOG'
+    contract.m_currency = 'USD'
+    contract.m_secType = 'STK'
+    contract.m_exchange = 'SMART'
+    #contract.m_strike = 58.5
+    #contract.m_multiplier = 100
+    #contract.m_includeExpired = False
+    
+    #contract.m_right = 'P'
+    #contract.m_expiry = '20151120'    
+    c.get_command_handler().reqMktData(contract)
+    contract = Contract()
+    contract.m_symbol = 'EWT'
+    contract.m_currency = 'USD'
+    contract.m_secType = 'STK'
+    contract.m_exchange = 'SMART'    
+    c.get_command_handler().reqMktData(contract)
+    sleep(1)
+    c.get_command_handler().gw_req_subscriptions()
+    
+    sleep(10)
+    c.disconnect()
+
+
+def test5():
+    print '******************************** TEST 5'
+    c = SampleClient(host, port, 'SampleClient-777')
+    c.connect()
+    #c.get_command_handler().reqIds()
+    c.get_command_handler().gw_req_subscriptions()
+    c.get_command_handler().reqExecutions()
+    sleep(8)
+    c.disconnect()
+    
+    
+def test6():
+    
+    contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20151127', 0, '')
+    #contractTuple = ('VMW', 'STK', 'SMART', 'USD', '', 0, '')
+    contract = ContractHelper.makeContract(contractTuple)  
+    oc = OptionsChain('test6')
+    
+    oc.set_underlying(contract)
+    oc.set_option_structure(contract, 200, 50, 0.005, 0.003, '20151127')
+    oc.build_chain(22300, 0.1)    
+    c = SampleClient(host, port, 'SampleClient-777')
+    c.connect()
+    #c.get_command_handler().reqIds()
+    c.get_command_handler().gw_req_subscriptions()
+    
+
+    c.get_command_handler().reqMktData(contract)
+    for ct in oc.get_option_chain():    
+        c.get_command_handler().reqMktData(ct.get_contract())
+        print ContractHelper.object2kvstring(ct.get_contract())
+    sleep(8)
+    c.disconnect()
+        
+        
+def test7():
+    contractTuple = ('VMW', 'STK', 'SMART', 'USD', '', 0, '')
+    contract = ContractHelper.makeContract(contractTuple)  
+    oc = OptionsChain('t7')
+    
+    
+    oc.set_underlying(contract)
+       
+    oc.set_option_structure(contract, 0.5, 100, 0.0012, 0.0328, '20151211')
+    oc.build_chain(59.3, 0.08, 0.22)            
+    
+    c = SampleClient(host, port, 'SampleClient-777')
+    c.connect()
+    #c.get_command_handler().reqIds()
+    c.get_command_handler().gw_req_subscriptions()
+    
+
+    c.get_command_handler().reqMktData(contract)
+    for ct in oc.get_option_chain():    
+        c.get_command_handler().reqMktData(ct.get_contract())
+        print ContractHelper.object2kvstring(ct.get_contract())
+    sleep(3)
+    c.disconnect()
+    
+def test8():    
+#     c = SampleClient(host, port, 'SampleClient-777')
+#     c.connect()
+    
+    consumer = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.oceEvents)] , \
+                                   metadata_broker_list=['%s:%s' % (host, port)],\
+                                   client_id = 'test8',\
+                                   group_id = 'epc.group',\
+                                   auto_commit_enable=True,\
+                                   consumer_timeout_ms = 2000,\
+                                   auto_commit_interval_ms=30 * 1000,\
+                                   auto_offset_reset='smallest')    
+    
+#     c.get_command_handler().gw_req_subscriptions()
+    for message in consumer:
+          
+
+    
+        print ("received %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+                                     message.offset, message.key,
+                                     message.value))    
+    
+    print 'here'
+#     c.disconnect()
+
+
+def on_analytics(msg):
+    print msg
+    kv = json.loads(msg)
+    print 'x=%s imvol=%0.2f theta=%0.2f' % (kv['contract']['m_strike'], kv['analytics']['imvol'],kv['analytics']['theta'])
+
+def test9():
+    al = AnalyticsListener(host, port, 'al')
+    al.registerAll([on_analytics])
+    al.start()
+            
+if __name__ == '__main__':
+    """ 
+        this sample demonstrates how to use SimpleTWSClient
+        to connect to IB TWS gateway 
+        
+        
+        
+        usage scenarios:
+        
+        case 1
+        re-use SimpleTWSClient object
+        register to listen for all messages
+        perform processing within the callback function
+        
+        case 2
+        inherit SimpleTWSClient class
+        override event callback functions to use
+        each function associates with a specific message type
+        
+    """
+    if len(sys.argv) != 2:
+        print("Usage: %s <test case #>" % sys.argv[0])
+        exit(-1)    
+
+    choice= sys.argv[1]
+           
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s')
+    
+    host = 'vsu-01'
+    port = 9092
+
+    print 'choice: %s' % choice
+    test9()
+    #test8()
+#     if choice == '2': 
+#         
+#         test2()
+#     else:
+#         
+#         test3()
+    
+    

comms/tws_client.py

@@ -0,0 +1,590 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+import json
+import logging
+import ConfigParser
+from time import sleep
+import time, datetime
+from threading import Lock
+from kafka.client import KafkaClient
+from kafka import KafkaConsumer
+from kafka.producer import SimpleProducer
+from kafka.common import LeaderNotAvailableError, ConsumerTimeout
+import threading
+
+from misc2.helpers import ContractHelper, OrderHelper, ExecutionFilterHelper
+import uuid
+
+
+from tws_protocol_helper import TWS_Protocol, Message
+
+        
+class TWS_client_base_app(threading.Thread):
+    
+    producer = None
+    consumer = None
+    command_handler = None
+    stop_consumer = False
+    
+    reg_all_callbacks = []
+#    reg_event_func_map = {}
+
+
+    def __init__(self, host, port, id=None):
+
+        super(TWS_client_base_app, self).__init__()
+        client = KafkaClient('%s:%s' % (host, port))
+        self.producer = SimpleProducer(client, async=False)
+        
+
+ 
+        # consumer_timeout_ms must be set - this allows the consumer an interval to exit from its blocking loop
+        self.consumer = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.topicEvents) + list(TWS_Protocol.gatewayEvents)] , \
+                                       metadata_broker_list=['%s:%s' % (host, port)],\
+                                       client_id = str(uuid.uuid1()) if id == None else id,\
+                                       group_id = 'epc.group',\
+                                       auto_commit_enable=True,\
+                                       consumer_timeout_ms = 2000,\
+                                       auto_commit_interval_ms=30 * 1000,\
+                                       auto_offset_reset='largest') # discard old ones
+        
+        self.reset_message_offset()
+        
+        #self.consumer.set_topic_partitions(('gw_subscriptions', 0, 114,),('tickPrice', 0, 27270,))
+        self.command_handler= TWS_server_wrapper(self.producer)
+        
+
+    def reset_message_offset(self):
+        # 90 is a magic number or don't care (max_num_offsets)
+        topic_offsets =  map(lambda topic: (topic, self.consumer.get_partition_offsets(topic, 0, -1, 90)), TWS_Protocol.topicEvents + TWS_Protocol.gatewayEvents)
+        topic_offsets = filter(lambda x: x <> None, map(lambda x: (x[0], x[1][1], max(x[1][0], 0)) if len(x[1]) > 1 else None, topic_offsets))
+        logging.info("TWS_client_base_app: topic offset dump ------:")
+        logging.info (topic_offsets)
+        logging.info('TWS_client_base_app set topic offset to the latest point\n%s' % (''.join('%s,%s,%s\n' % (x[0], x[1], x[2]) for x in topic_offsets)))
+        
+        # the set_topic_partitions call clears out all previous settings when starts
+        # therefore it's not possible to do something like this:
+        # self.consumer.set_topic_partitions(('gw_subscriptions', 0, 114,)
+        # self.consumer.set_topic_partitions(('tickPrice', 0, 27270,))
+        # as the second call will wipe out whatever was done previously
+        self.consumer.set_topic_partitions(*topic_offsets)        
+        
+
+    def get_producer(self):
+        return self.producer
+                            
+    def run(self):
+
+            
+#             for message in self.consumer:
+#                  
+#                 logging.info("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+#                                              message.offset, message.key,
+#                                              message.value))
+#      
+#                 print ("received %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+#                                              message.offset, message.key,
+#                                              message.value))
+#                  
+#                 #self.on_tickPrice(message.value)
+#                 getattr(self, message.topic, None)(message.value)
+#                 # send message to the reg callback func pointers
+#                 # the message is turned into IB compatible type before firing the callbacks
+#                 [f(self.convertItemsToIBmessage(message.value)) for f in self.reg_all_callbacks]
+
+            logging.info ('TWS_client_base_app: consumer_timeout_ms = %d' % self.consumer._config['consumer_timeout_ms']) 
+ 
+            # keep running until someone tells us to stop
+            while self.stop_consumer == False:
+            #while True:
+  
+                    try:
+                        # the next() function runs an infinite blocking loop
+                        # it will raise a consumertimeout if no message is received after a pre-set interval   
+                        message = self.consumer.next()
+                        
+                        
+                        logging.debug("TWS_client_base_app: %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+                                                     message.offset, message.key,
+                                                     message.value))
+                        
+                        kvmessage = self.convertItemsToIBmessage(message.value) 
+                        
+                        
+                        getattr(self, message.topic, None)(kvmessage)
+                        # send message to the reg callback func pointers
+                        # the message is trasnformed into an IB compatible msg before firing the callbacks
+                        [f(kvmessage) for f in self.reg_all_callbacks]                        
+                        
+                        #self.consumer.task_done(message)
+                        
+                    except ConsumerTimeout:
+                        logging.info('TWS_client_base_app run: ConsumerTimeout. Check new message in the next round...')
+                                        
+                  
+                
+
+    def stop(self):
+        logging.info('TWS_client_base_app: --------------- stopping consumer')
+        self.stop_consumer = True
+        
+
+    def registerAll(self, funcs):
+        [self.reg_all_callbacks.append(f) for f in funcs]
+        
+#     def register(self, event, func):
+#         if event in TWS_Protocol.topicEvents:
+#             self.reg_event_func_map[event] = [func] if self.reg_event_func_map[event] is None else self.reg_event_func_map[event].append(func)
+#         else:
+#             # raise exception
+#             pass 
+
+    def handlemessage(self, msg_name, mapping):
+
+        items = list(mapping.items())
+        items.sort()
+        print(('### %s' % (msg_name, )))
+        for k, v in items:
+            print(('    %s:%s' % (k, v)))
+
+    
+    def ascii_encode_dict(self, data):
+        ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
+        return dict(map(ascii_encode, pair) for pair in data.items())
+
+
+    def convertItemsToIBmessage(self, items):
+        # convert into our version of Message
+        try:
+            items = json.loads(items, object_hook=self.ascii_encode_dict)
+            if 'contract' in items:
+                items['contract'] = ContractHelper.kv2contract(items['contract'])
+            del(items['self'])
+        except (KeyError, ):
+            pass        
+        return Message(**items)
+    
+    def get_command_handler(self):
+        return self.command_handler
+
+    def accountDownloadEnd(self, items):
+        self.handlemessage("accountDownloadEnd", items)
+
+    def execDetailsEnd(self, items):
+        self.handlemessage("execDetailsEnd", items)
+
+    def updateAccountTime(self, items):
+        self.handlemessage("updateAccountTime", items)
+
+    def deltaNeutralValidation(self, items):
+        self.handlemessage("deltaNeutralValidation", items)
+
+    def orderStatus(self, items):
+        self.handlemessage("orderStatus", items)
+
+    def updateAccountValue(self, items):
+        self.handlemessage("updateAccountValue", items)
+
+    def historicalData(self, items):
+        self.handlemessage("historicalData", items)
+
+    def openOrderEnd(self, items):
+        self.handlemessage("openOrderEnd", items)
+
+    def updatePortfolio(self, items):
+        self.handlemessage("updatePortfolio", items)
+
+    def managedAccounts(self, items):
+        self.handlemessage("managedAccounts", items)
+
+    def contractDetailsEnd(self, items):
+        self.handlemessage("contractDetailsEnd", items)
+
+    def positionEnd(self, items):
+        self.handlemessage("positionEnd", items)
+
+    def bondContractDetails(self, items):
+        self.handlemessage("bondContractDetails", items)
+
+    def accountSummary(self, items):
+        self.handlemessage("accountSummary", items)
+
+    def updateNewsBulletin(self, items):
+        self.handlemessage("updateNewsBulletin", items)
+
+    def scannerParameters(self, items):
+        self.handlemessage("scannerParameters", items)
+
+    def tickString(self, items):
+        self.handlemessage("tickString", items)
+
+    def accountSummaryEnd(self, items):
+        self.handlemessage("accountSummaryEnd", items)
+
+    def scannerDataEnd(self, items):
+        self.handlemessage("scannerDataEnd", items)
+
+    def commissionReport(self, items):
+        self.handlemessage("commissionReport", items)
+
+    def error(self, items):
+        self.handlemessage("error", items)
+
+    def tickGeneric(self, items):
+        self.handlemessage("tickGeneric", items)
+
+    def tickPrice(self, items):
+        self.handlemessage("tickPrice", items)
+
+    def nextValidId(self, items):
+        self.handlemessage("nextValidId", items)
+
+    def openOrder(self, items):
+        self.handlemessage("openOrder", items)
+
+    def realtimeBar(self, items):
+        self.handlemessage("realtimeBar", items)
+
+    def contractDetails(self, items):
+        self.handlemessage("contractDetails", items)
+
+    def execDetails(self, items):
+        self.handlemessage("execDetails", items)
+
+    def tickOptionComputation(self, items):
+        self.handlemessage("tickOptionComputation", items)
+
+    def updateMktDepth(self, items):
+        self.handlemessage("updateMktDepth", items)
+
+    def scannerData(self, items):
+        self.handlemessage("scannerData", items)
+
+    def currentTime(self, items):
+        self.handlemessage("currentTime", items)
+
+    def error_0(self, items):
+        self.handlemessage("error_0", items)
+
+    def error_1(self, items):
+        self.handlemessage("error_1", items)
+
+    def tickSnapshotEnd(self, items):
+        self.handlemessage("tickSnapshotEnd", items)
+
+    def tickSize(self, items):
+        self.handlemessage("tickSize", items)
+
+    def receiveFA(self, items):
+        self.handlemessage("receiveFA", items)
+
+    def connectionClosed(self, items):
+        self.handlemessage("connectionClosed", items)
+
+    def position(self, items):
+        self.handlemessage("position", items)
+
+    def updateMktDepthL2(self, items):
+        self.handlemessage("updateMktDepthL2", items)
+
+    def fundamentalData(self, items):
+        self.handlemessage("fundamentalData", items)
+
+    def tickEFP(self, items):
+        self.handlemessage("tickEFP", items)
+
+
+###########################################################################3333
+#   Gateway respond events
+#
+#
+
+    def gw_subscriptions(self, items):
+        self.handlemessage("gw_subscriptions", items)
+
+
+#     def on_tickPrice(self, tickerId, field, price, canAutoExecute):
+#         self.handlemessage('tickPrice', vars())
+# 
+#     def on_tickSize(self, tickerId, field, size):
+#         self.handlemessage('tickSize', vars())
+# 
+#     def on_tickOptionComputation(self, tickerId, field, impliedVol, delta, optPrice, pvDividend, gamma, vega, theta, undPrice):
+#         self.handlemessage('tickOptionComputation', vars())
+# 
+#     def on_tickGeneric(self, tickerId, tickType, value):
+#         self.handlemessage('tickGeneric', vars())
+# 
+#     def on_tickString(self, tickerId, tickType, value):
+#         self.handlemessage('tickString', vars())
+# 
+#     def on_tickEFP(self, tickerId, tickType, basisPoints, formattedBasisPoints, impliedFuture, holdDays, futureExpiry, dividendImpact, dividendsToExpiry):
+#         self.handlemessage('tickEFP', vars())
+# 
+#     def on_orderStatus(self, orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice, clientId, whyHeId):
+#         self.handlemessage('orderStatus', vars())
+# 
+#     def on_openOrder(self, orderId, contract, order, state):
+#         self.handlemessage('openOrder', vars())
+# 
+#     def on_openOrderEnd(self):
+#         self.handlemessage('openOrderEnd', vars())
+# 
+#     def on_updateAccountValue(self, key, value, currency, accountName):
+#         self.handlemessage('updateAccountValue', vars())
+# 
+#     def on_updatePortfolio(self, contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName):
+#         self.handlemessage('updatePortfolio', vars())
+# 
+#     def on_updateAccountTime(self, timeStamp):
+#         self.handlemessage('updateAccountTime', vars())
+# 
+#     def on_accountDownloadEnd(self, accountName):
+#         self.handlemessage('accountDownloadEnd', vars())
+# 
+#     def on_nextValidId(self, orderId):
+#         self.handlemessage('nextValidId', vars())
+# 
+#     def on_contractDetails(self, reqId, contractDetails):
+#         self.handlemessage('contractDetails', vars())
+# 
+#     def on_contractDetailsEnd(self, reqId):
+#         self.handlemessage('contractDetailsEnd', vars())
+# 
+#     def on_bondContractDetails(self, reqId, contractDetails):
+#         self.handlemessage('bondContractDetails', vars())
+# 
+#     def on_execDetails(self, reqId, contract, execution):
+#         self.handlemessage('execDetails', vars())
+# 
+#     def on_execDetailsEnd(self, reqId):
+#         self.handlemessage('execDetailsEnd', vars())
+# 
+#     def on_connectionClosed(self):
+#         self.handlemessage('connectionClosed', {})
+# 
+#     def on_error(self, id=None, errorCode=None, errorMsg=None):
+#         self.handlemessage('error', vars())
+# 
+#     def on_error_0(self, strvalue=None):
+#         self.handlemessage('error_0', vars())
+# 
+#     def on_error_1(self, id=None, errorCode=None, errorMsg=None):
+#         self.handlemessage('error_1', vars())
+# 
+#     def on_updateMktDepth(self, tickerId, position, operation, side, price, size):
+#         self.handlemessage('updateMktDepth', vars())
+# 
+#     def on_updateMktDepthL2(self, tickerId, position, marketMaker, operation, side, price, size):
+#         self.handlemessage('updateMktDepthL2', vars())
+# 
+#     def on_updateNewsBulletin(self, msgId, msgType, message, origExchange):
+#         self.handlemessage('updateNewsBulletin', vars())
+# 
+#     def on_managedAccounts(self, accountsList):
+#         self.handlemessage('managedAccounts', vars())
+# 
+#     def on_receiveFA(self, faDataType, xml):
+#         self.handlemessage('receiveFA', vars())
+# 
+#     def on_historicalData(self, reqId, date, open, high, low, close, volume, count, WAP, hasGaps):
+#         self.handlemessage('historicalData', vars())
+# 
+#     def on_scannerParameters(self, xml):
+#         self.handlemessage('scannerParameters', vars())
+# 
+#     def on_scannerData(self, reqId, rank, contractDetails, distance, benchmark, projection, legsStr):
+#         self.handlemessage('scannerData', vars())
+# 
+# 
+#     def on_commissionReport(self, commissionReport):
+#         self.handlemessage('commissionReport', vars())
+# 
+# 
+#     def on_currentTime(self, time):
+#         self.handlemessage('currentTime', vars())
+# 
+#     def on_deltaNeutralValidation(self, reqId, underComp):
+#         self.handlemessage('deltaNeutralValidation', vars())
+# 
+# 
+#     def on_fundamentalData(self, reqId, data):
+#         self.handlemessage('fundamentalData', vars())
+# 
+#     def on_marketDataType(self, reqId, marketDataType):
+#         self.handlemessage('marketDataType', vars())
+# 
+# 
+#     def on_realtimeBar(self, reqId, time, open, high, low, close, volume, wap, count):
+#         self.handlemessage('realtimeBar', vars())
+# 
+#     def on_scannerDataEnd(self, reqId):
+#         self.handlemessage('scannerDataEnd', vars())
+# 
+# 
+# 
+#     def on_tickSnapshotEnd(self, reqId):
+#         self.handlemessage('tickSnapshotEnd', vars())
+# 
+# 
+#     def on_position(self, account, contract, pos, avgCost):
+#         self.handlemessage('position', vars())
+# 
+#     def on_positionEnd(self):
+#         self.handlemessage('positionEnd', vars())
+# 
+#     def on_accountSummary(self, reqId, account, tag, value, currency):
+#         self.handlemessage('accountSummary', vars())
+# 
+#     def on_accountSummaryEnd(self, reqId):
+#         self.handlemessage('accountSummaryEnd', vars())
+
+
+
+
+
+
+
+
+class TWS_server_wrapper():
+    
+    producer = None
+    def __init__(self, producer):
+        self.producer = producer
+
+
+    
+    def reqOpenOrders(self):
+        self.post_msg('reqOpenOrders', '')
+    
+    
+    def reqIds(self):
+        self.post_msg('reqIds', '')
+    
+    def reqNewsBulletins(self):
+        logging.error('reqNewsBulletins: NOT IMPLEMENTED')
+    
+    def cancelNewsBulletins(self):
+        logging.error('cancelNewsBulletins: NOT IMPLEMENTED')
+
+    
+    def setServerLogLevel(self):
+        logging.error('setServerLogLevel: NOT IMPLEMENTED')
+
+    
+    def reqAutoOpenOrders(self):
+        logging.error('reqAutoOpenOrders: NOT IMPLEMENTED')
+
+    
+    def reqAllOpenOrders(self):
+        logging.error('reqAllOpenOrders: NOT IMPLEMENTED')
+
+    
+    def reqManagedAccts(self):
+        logging.error('reqManagedAccts: NOT IMPLEMENTED')
+
+    
+    def requestFA(self):
+        logging.error('requestFA: NOT IMPLEMENTED')
+
+    
+    def reqPositions(self):
+        self.post_msg('reqPositions', '')
+        
+    def reqHistoricalData(self):
+        logging.error('reqHistoricalData: NOT IMPLEMENTED')
+        
+
+    def reqAccountUpdates(self):
+        
+        self.post_msg('reqAccountUpdates', '1')
+
+    def reqExecutions(self, exec_filter=None):
+        
+        self.post_msg('reqExecutions', ExecutionFilterHelper.object2kvstring(exec_filter) if exec_filter <> None else '')
+
+
+    def reqMktData(self, contract):
+        #self.post_msg('reqMktData', ContractHelper.contract2kvstring(contract))
+        self.post_msg('reqMktData', ContractHelper.object2kvstring(contract))
+        
+    def reqAccountSummary(self, reqId, group, tags):
+        self.post_msg('reqAccountSummary', json.dumps([reqId, group, tags]))
+    
+    def placeOrder(self, id, contract, order):
+        self.post_msg('placeOrder', json.dumps([id, ContractHelper.contract2kvstring(contract), OrderHelper.object2kvstring(order)]))
+    
+        
+    def post_msg(self, topic, msg):
+        logging.info('post_msg sending request to gateway: %s[%s]' % (topic,msg))
+        self.producer.send_messages(topic, msg)
+
+
+#############################################################33
+#  Gateway methods
+
+    def gw_req_subscriptions(self):
+        self.post_msg('gw_req_subscriptions', '')
+
+
+
+
+class SimpleTWSClient(TWS_client_base_app):
+    
+    def __init__(self, host, port, id=None):
+        super(SimpleTWSClient, self).__init__(host, port, id)
+        logging.info('SimpleTWSClient client id=%s' % id)
+    
+    def connect(self):
+        self.start()
+    
+    def disconnect(self):
+        logging.info ('SimpleTWSClient: received disconnect. asking base class consumer to stop...')
+        self.stop()
+    
+
+
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0: 
+        raise ValueError, "Failed to open config file" 
+      
+    logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s %(levelname)s %(message)s')
+
+    host = config.get("epc", "kafka.host").strip('"').strip("'")
+    port = config.get("epc", "kafka.port")
+
+
+    e = SimpleTWSClient(host, port)
+    #e.registerAll([e.on_ib_message])
+    e.start()
+    #e.command_handler.reqAccountUpdates()
+    #e.command_handler.reqExecutions()
+    #USD,CASH,IDEALPRO,JPY,,0,
+    contractTuple = ('USD', 'CASH', 'IDEALPRO', 'JPY', '', 0, '')
+    c = ContractHelper.makeContract(contractTuple)   
+    
+    #e.command_handler.reqMktData(c)
+    contractTuple = ('USO', 'STK', 'SMART', 'USD', '', 0, '')
+    c = ContractHelper.makeContract(contractTuple)   
+    #e.command_handler.reqMktData(c)
+    e.get_command_handler().reqPositions()
+#    print dummy()
+#     kwargs = {"arg3": 3, "arg2": "two","arg1":5}
+#     m = Message(**kwargs)
+#     print m.items()
+#     print m.arg3
+    
+    
+
+    

comms/tws_gateway.py

@@ -0,0 +1,787 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##
+# This script is an exmple of using the generated code within IbPy in
+# the same manner as the Java code.  We subclass EWrapper and give an
+# instance of the wrapper to an EClientSocket.
+##
+
+
+import sys
+from time import sleep, strftime
+import time, datetime
+import ConfigParser
+from optparse import OptionParser
+import logging
+import thread
+import threading
+import traceback
+import json
+from threading import Lock
+from ib.ext.Contract import Contract
+from ib.ext.EWrapper import EWrapper
+from ib.ext.EClientSocket import EClientSocket
+from ib.ext.ExecutionFilter import ExecutionFilter
+from ib.ext.Execution import Execution
+from ib.ext.OrderState import OrderState
+from ib.ext.Order import Order
+
+from kafka.client import KafkaClient
+from kafka import KafkaConsumer
+from kafka.producer import SimpleProducer
+from kafka.common import LeaderNotAvailableError
+
+from misc2.helpers import ContractHelper, OrderHelper, ExecutionFilterHelper
+from comms.ib_heartbeat import IbHeartBeat
+from tws_protocol_helper import TWS_Protocol 
+
+import redis
+         
+
+        
+        
+class TWS_event_handler(EWrapper):
+
+    TICKER_GAP = 1000
+    producer = None
+    
+    def __init__(self, host, port):
+        
+        client = KafkaClient('%s:%s' % (host, port))
+        self.producer = SimpleProducer(client, async=False)    
+        logging.info('TWS_event_handler: __init__ Creating kafka client producer at %s:%s' % (host, port))
+ 
+ 
+    def serialize_vars_to_dict(self, message, mapping, source='IB'):
+        def create_kmessage(items):
+            d = {}
+            for k,v in items:
+                #print k, v, type(v)
+                #if type(v) in [Contract, Execution, ExecutionFilter, OrderState, Order, CommissionReport]:
+                if 'ib.ext.' in str(type(v)):     
+                    d[k] = v.__dict__
+                else:
+                    d[k] = v
+    
+               
+            d['ts'] = time.time()
+            d['typeName'] = message
+            d['source'] = source
+            return d
+        
+        
+        try:
+            del(mapping['self'])
+        except (KeyError, ):
+            pass
+        items = list(mapping.items())
+        return create_kmessage(items)
+ 
+    def broadcast_event(self, message, mapping, source='IB'):
+
+        try:
+            dict = self.serialize_vars_to_dict(message, mapping, source)     
+            if message == 'gw_subscriptions':   
+                logging.info('TWS_event_handler: broadcast event: %s [%s]' % (dict['typeName'], dict))
+            self.producer.send_messages(message, json.dumps(dict))    
+        except:
+            logging.error('broadcast_event: exception while encoding IB event to client:  [%s]' % message)
+            logging.error(traceback.format_exc())
+            
+            #
+            # try to broadcast the message a 2nd time
+            # no catch if fails again
+            if message == 'gw_subscriptions':   
+                sleep(2)
+                logging.info('TWS_event_handler: Retry once broadcasting gw_subscription ' % (dict['typeName'], dict))
+                self.producer.send_messages(message, json.dumps(dict))    
+            
+            
+
+    
+    def tick_process_message(self, items):
+        
+        t = {}
+        t = items.copy()
+        # if the tickerId is in the snapshot range
+        # deduct the gap to derive the original tickerId
+        # --- check logic in subscription manager
+        if (t['tickerId']  >= TWS_event_handler.TICKER_GAP):
+            t['tickerId'] = t['tickerId']  - TWS_event_handler.TICKER_GAP
+        try:
+            del(t['self'])
+        except (KeyError, ):
+            pass          
+        
+        return t  
+            
+                
+    
+    def tickPrice(self, tickerId, field, price, canAutoExecute):
+        
+        self.broadcast_event('tickPrice', self.tick_process_message(vars()))
+
+    def tickSize(self, tickerId, field, size):
+        
+        self.broadcast_event('tickSize', self.tick_process_message(vars())) #vars())
+
+    def tickOptionComputation(self, tickerId, field, impliedVol, delta, optPrice, pvDividend, gamma, vega, theta, undPrice):
+        
+        #self.broadcast_event('tickOptionComputation', self.tick_process_message(vars())) #vars())
+        pass
+
+    def tickGeneric(self, tickerId, tickType, value):
+        #self.broadcast_event('tickGeneric', vars())
+        self.broadcast_event('tickGeneric', self.tick_process_message(vars())) #vars())
+
+    def tickString(self, tickerId, tickType, value):
+        #self.broadcast_event('tickString', vars())
+        self.broadcast_event('tickString', self.tick_process_message(vars())) #vars())
+
+    def tickEFP(self, tickerId, tickType, basisPoints, formattedBasisPoints, impliedFuture, holdDays, futureExpiry, dividendImpact, dividendsToExpiry):
+        self.broadcast_event('tickEFP', vars())
+
+    def orderStatus(self, orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice, clientId, whyHeId):
+        self.broadcast_event('orderStatus', vars())
+
+    def openOrder(self, orderId, contract, order, state):
+        self.broadcast_event('openOrder', vars())
+
+    def openOrderEnd(self):
+        self.broadcast_event('openOrderEnd', vars())
+
+    def updateAccountValue(self, key, value, currency, accountName):
+        self.broadcast_event('updateAccountValue', vars())
+
+    def updatePortfolio(self, contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName):
+        self.broadcast_event('updatePortfolio', vars())
+
+    def updateAccountTime(self, timeStamp):
+        self.broadcast_event('updateAccountTime', vars())
+
+    def accountDownloadEnd(self, accountName):
+        self.broadcast_event('accountDownloadEnd', vars())
+
+    def nextValidId(self, orderId):
+        self.broadcast_event('nextValidId', vars())
+
+    def contractDetails(self, reqId, contractDetails):
+        self.broadcast_event('contractDetails', vars())
+
+    def contractDetailsEnd(self, reqId):
+        self.broadcast_event('contractDetailsEnd', vars())
+
+    def bondContractDetails(self, reqId, contractDetails):
+        self.broadcast_event('bondContractDetails', vars())
+
+    def execDetails(self, reqId, contract, execution):
+        self.broadcast_event('execDetails', vars())
+
+    def execDetailsEnd(self, reqId):
+        self.broadcast_event('execDetailsEnd', vars())
+
+    def connectionClosed(self):
+        self.broadcast_event('connectionClosed', {})
+
+    def error(self, id=None, errorCode=None, errorMsg=None):
+        logging.error(self.serialize_vars_to_dict('error', vars()))
+        self.broadcast_event('error', vars())
+
+    def error_0(self, strvalue=None):
+        logging.error(self.serialize_vars_to_dict('error_0', vars()))
+        self.broadcast_event('error_0', vars())
+
+    def error_1(self, id=None, errorCode=None, errorMsg=None):
+        logging.error(self.serialize_vars_to_dict('error_1', vars()))        
+        self.broadcast_event('error_1', vars())
+
+    def updateMktDepth(self, tickerId, position, operation, side, price, size):
+        self.broadcast_event('updateMktDepth', vars())
+
+    def updateMktDepthL2(self, tickerId, position, marketMaker, operation, side, price, size):
+        self.broadcast_event('updateMktDepthL2', vars())
+
+    def updateNewsBulletin(self, msgId, msgType, message, origExchange):
+        self.broadcast_event('updateNewsBulletin', vars())
+
+    def managedAccounts(self, accountsList):
+        logging.info(self.serialize_vars_to_dict('managedAccounts', vars()))
+        self.broadcast_event('managedAccounts', vars())
+
+    def receiveFA(self, faDataType, xml):
+        self.broadcast_event('receiveFA', vars())
+
+    def historicalData(self, reqId, date, open, high, low, close, volume, count, WAP, hasGaps):
+        self.broadcast_event('historicalData', vars())
+
+    def scannerParameters(self, xml):
+        self.broadcast_event('scannerParameters', vars())
+
+    def scannerData(self, reqId, rank, contractDetails, distance, benchmark, projection, legsStr):
+        self.broadcast_event('scannerData', vars())
+
+
+    def commissionReport(self, commissionReport):
+        self.broadcast_event('commissionReport', vars())
+
+
+    def currentTime(self, time):
+        self.broadcast_event('currentTime', vars())
+
+    def deltaNeutralValidation(self, reqId, underComp):
+        self.broadcast_event('deltaNeutralValidation', vars())
+
+
+    def fundamentalData(self, reqId, data):
+        self.broadcast_event('fundamentalData', vars())
+
+    def marketDataType(self, reqId, marketDataType):
+        self.broadcast_event('marketDataType', vars())
+
+
+    def realtimeBar(self, reqId, time, open, high, low, close, volume, wap, count):
+        self.broadcast_event('realtimeBar', vars())
+
+    def scannerDataEnd(self, reqId):
+        self.broadcast_event('scannerDataEnd', vars())
+
+
+    def tickSnapshotEnd(self, reqId):
+        self.broadcast_event('tickSnapshotEnd', vars())
+
+
+    def position(self, account, contract, pos, avgCost):
+        self.broadcast_event('position', vars())
+
+    def positionEnd(self):
+        self.broadcast_event('positionEnd', vars())
+
+    def accountSummary(self, reqId, account, tag, value, currency):
+        self.broadcast_event('accountSummary', vars())
+
+    def accountSummaryEnd(self, reqId):
+        self.broadcast_event('accountSummaryEnd', vars())
+
+
+
+class TWS_gateway(threading.Thread):
+
+    # config
+    config = None
+    # redis connection
+    rs = None
+
+    
+    # channel clients' requests to IB/TWS
+    cli_request_handler = None
+
+    # manage conID / contracts mapping
+    contract_subscription_mgr = None  
+    
+    connection = None
+    
+    # handler to process incoming IB/TWS messages and echo back to clients  
+    tws_event_handler = None
+    
+    # monitor IB connection / heart beat
+    ibh = None
+    tlock = None
+    ib_conn_status = None
+    ib_order_transmit = False
+    
+    
+    def __init__(self, host, port, clientId, kafka_host, kafka_port, config):
+        super(TWS_gateway, self).__init__()
+        self.config = config
+        self.host = host
+        self.port = port
+        self.clientId = clientId
+        self.ib_order_transmit = config.get("tws_gateway", "tws_gateway.order_transmit").strip('"').strip("'") if \
+                                        config.get("tws_gateway", "tws_gateway.order_transmit").strip('"').strip("'") <> None\
+                                        else False
+        
+        logging.info('starting up TWS_gateway...')
+        logging.info('Order straight through (no-touch) flag = %s' % ('True' if self.ib_order_transmit == True else 'False'))
+        
+        logging.info('connecting to Redis server...')
+        self.initialize_redis(config)
+        
+        logging.info('starting up TWS_event_handler...')
+        self.tws_event_handler = TWS_event_handler(kafka_host, kafka_port)
+        
+        logging.info('starting up IB EClientSocket...')
+        self.connection = EClientSocket(self.tws_event_handler)
+        
+        
+        logging.info('starting up client request handler - kafkaConsumer...')
+        self.cli_request_handler = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.topicMethods) + list(TWS_Protocol.gatewayMethods) ], \
+                                   metadata_broker_list=['%s:%s' % (kafka_host, kafka_port)],\
+                                   group_id = 'epc.tws_gateway',\
+                                   auto_commit_enable=True,\
+                                   auto_commit_interval_ms=30 * 1000,\
+                                   auto_offset_reset='largest') # discard old ones
+        
+        self.reset_message_offset()
+        
+
+
+        
+        
+
+
+         
+        
+
+        if not self.eConnect():
+            logging.error('TWS_gateway: unable to establish connection to IB %s:%d' % (self.host, self.port))
+            sys.exit(-1)
+        else:
+            # start heart beat monitor
+            logging.info('starting up IB heart beat monitor...')
+            self.tlock = Lock()
+            self.ibh = IbHeartBeat(config)
+            self.ibh.register_listener([self.on_ib_conn_broken])
+            self.ibh.run()  
+
+
+        logging.info('starting up subscription manager...')
+        self.initialize_subscription_mgr()
+
+
+
+    def initialize_subscription_mgr(self):
+        
+        self.contract_subscription_mgr = SubscriptionManager(self)
+        self.contract_subscription_mgr.register_persistence_callback(self.persist_subscriptions)
+        key = self.config.get("tws_gateway",  "subscription_manager.subscriptions.redis_key").strip('"').strip("'")
+        if self.rs.get(key):
+            #contracts = map(lambda x: ContractHelper.kvstring2contract(x), json.loads(self.rs.get(key)))
+            contracts = map(lambda x: ContractHelper.kvstring2object(x, Contract), json.loads(self.rs.get(key)))
+            self.contract_subscription_mgr.load_subscription(contracts)
+        
+
+    def persist_subscriptions(self, contracts):
+         
+        key = self.config.get("tws_gateway",  "subscription_manager.subscriptions.redis_key").strip('"').strip("'")
+        #cs = json.dumps(map(lambda x: ContractHelper.contract2kvstring(x) if x <> None else None, contracts))
+        cs = json.dumps(map(lambda x: ContractHelper.object2kvstring(x) if x <> None else None, contracts))
+        logging.debug('Tws_gateway: updating subscription table to redis store %s' % cs)
+        self.rs.set(key, cs)
+
+
+    def initialize_redis(self, config):
+        r_host = config.get("redis", "redis.server").strip('"').strip("'")
+        r_port = config.get("redis", "redis.port")
+        r_db = config.get("redis", "redis.db")     
+
+        self.rs = redis.Redis(r_host, r_port, r_db)
+        try:
+            self.rs.client_list()
+        except redis.ConnectionError:
+            logging.error('TWS_gateway: unable to connect to redis server using these settings: %s port:%d db:%d' % (r_host, r_port, r_db))
+            logging.error('aborting...')
+            sys.exit(-1)
+            
+
+    def reset_message_offset(self):
+        topic_offsets =  map(lambda topic: (topic, self.cli_request_handler.get_partition_offsets(topic, 0, -1, 999)), TWS_Protocol.topicMethods + TWS_Protocol.gatewayMethods)
+        topic_offsets = filter(lambda x: x <> None, map(lambda x: (x[0], x[1][1], x[1][0]) if len(x[1]) > 1 else None, topic_offsets))
+        logging.info('TWS_gateway set topic offset to the latest point\n%s' % (''.join('%s,%s,%s\n' % (x[0], x[1], x[2]) for x in topic_offsets)))
+
+        # the set_topic_partitions method clears out all previous settings when executed
+        # therefore it's not possible to call the function multiple times:
+        # self.consumer.set_topic_partitions(('gw_subscriptions', 0, 114,)
+        # self.consumer.set_topic_partitions(('tickPrice', 0, 27270,))
+        # as the second call will wipe out whatever was done previously
+
+        self.cli_request_handler.set_topic_partitions(*topic_offsets)
+        
+        
+        
+
+
+
+    def run(self):
+
+
+        for message in self.cli_request_handler:
+             
+            logging.info("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+                                         message.offset, message.key,
+                                         message.value))
+ 
+#             print ("TWS_gateway: received client request %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+#                                          message.offset, message.key,
+#                                          message.value))
+             
+
+            getattr(self, message.topic, None)(message.value)
+            #self.cli_request_handler.task_done(message)
+
+
+    def on_ib_conn_broken(self, msg):
+        logging.error('TWS_gateway: detected broken IB connection!')
+        self.ib_conn_status = 'ERROR'
+        self.tlock.acquire() # this function may get called multiple times
+        try:                 # block until another party finishes executing
+            if self.ib_conn_status == 'OK': # check status
+                return                      # if already fixed up while waiting, return 
+            
+            self.eDisconnect()
+            self.eConnect()
+            while not self.connection.isConnected():
+                logging.error('TWS_gateway: attempt to reconnect...')
+                self.eConnect()
+                sleep(2)
+            
+            # we arrived here because the connection has been restored
+            # resubscribe tickers again!
+            logging.info('TWS_gateway: IB connection restored...resubscribe contracts')
+            self.contract_subscription_mgr.force_resubscription()             
+            
+            
+        finally:
+            self.tlock.release()          
+        
+
+    
+    def eConnect(self):
+        logging.info('TWS_gateway - eConnect. Connecting to %s:%s App Id: %s' % (self.host, self.port, self.clientId))
+        self.connection.eConnect(self.host, self.port, self.clientId)
+        return self.connection.isConnected()
+
+    
+    def reqAccountUpdates(self, value=None):
+        logging.info('TWS_gateway - reqAccountUpdates value=%s' % value)
+        self.connection.reqAccountUpdates(1, '')
+
+    def reqAccountSummary(self, value):
+        logging.info('TWS_gateway - reqAccountSummary value=%s' % value)
+        
+        vals = map(lambda x: x.encode('ascii') if isinstance(x, unicode) else x, json.loads(value))
+        self.connection.reqAccountSummary(vals[0], vals[1], vals[2])
+        
+    def reqOpenOrders(self, value=None):
+        self.connection.reqOpenOrders()
+
+    def reqPositions(self, value=None):
+        self.connection.reqPositions()
+        
+        
+    def reqExecutions(self, value):
+        try:
+            filt = ExecutionFilter() if value == '' else ExecutionFilterHelper.kvstring2object(value, ExecutionFilter)
+            self.connection.reqExecutions(0, filt)
+        except:
+            logging.error(traceback.format_exc())
+
+    
+    def reqIds(self, value=None):
+        self.connection.reqIds(1)
+
+    
+    def reqNewsBulletins(self):
+        self.connection.reqNewsBulletins(1)
+
+    
+    def cancelNewsBulletins(self):
+        self.connection.cancelNewsBulletins()
+
+    
+    def setServerLogLevel(self):
+        self.connection.setServerLogLevel(3)
+
+    
+    def reqAutoOpenOrders(self):
+        self.connection.reqAutoOpenOrders(1)
+
+    
+    def reqAllOpenOrders(self):
+        self.connection.reqAllOpenOrders()
+
+    
+    def reqManagedAccts(self):
+        self.connection.reqManagedAccts()
+
+    
+    def requestFA(self):
+        self.connection.requestFA(1)
+
+    
+    def reqMktData(self, sm_contract):
+        logging.info('TWS Gateway received reqMktData request: %s' % sm_contract)
+        try:
+            #self.contract_subscription_mgr.reqMktData(ContractHelper.kvstring2contract(sm_contract))
+            self.contract_subscription_mgr.reqMktData(ContractHelper.kvstring2object(sm_contract, Contract))
+        except:
+            pass
+    
+    def reqHistoricalData(self):
+        contract = Contract()
+        contract.m_symbol = 'QQQQ'
+        contract.m_secType = 'STK'
+        contract.m_exchange = 'SMART'
+        endtime = strftime('%Y%m%d %H:%M:%S')
+        self.connection.reqHistoricalData(
+            tickerId=1,
+            contract=contract,
+            endDateTime=endtime,
+            durationStr='1 D',
+            barSizeSetting='1 min',
+            whatToShow='TRADES',
+            useRTH=0,
+            formatDate=1)
+
+    
+    def placeOrder(self, value=None):
+        logging.info('TWS_gateway - placeOrder value=%s' % value)
+        try:
+            vals = json.loads(value)
+        except ValueError:
+            logging.error('TWS_gateway - placeOrder Exception %s' % traceback.format_exc())
+            return
+        
+#        c = ContractHelper.kvstring2contract(vals[1])
+        o = OrderHelper.kvstring2object(vals[2], Order)
+        o.__dict__['transmit'] = self.ib_order_transmit
+#         print c.__dict__
+#         print o.__dict__
+#         print '---------------------'
+
+           
+        #self.connection.placeOrder(vals[0], ContractHelper.kvstring2contract(vals[1]), OrderHelper.kvstring2object(vals[2], Order))
+        self.connection.placeOrder(vals[0], ContractHelper.kvstring2object(vals[1], Contract), OrderHelper.kvstring2object(vals[2], Order))
+#        self.connection.placeOrder(orderId, contract, newOptOrder)
+        
+    def eDisconnect(self, value=None):
+        sleep(2)
+        self.connection.eDisconnect()
+ 
+ 
+####################################################################3
+#   Gateway commands
+    def gw_req_subscriptions(self, value=None):
+        
+        #subm = map(lambda i: ContractHelper.contract2kvstring(self.contract_subscription_mgr.handle[i]), range(len(self.contract_subscription_mgr.handle)))
+        #subm = map(lambda i: ContractHelper.object2kvstring(self.contract_subscription_mgr.handle[i]), range(len(self.contract_subscription_mgr.handle)))
+        subm = map(lambda i: (i, ContractHelper.object2kvstring(self.contract_subscription_mgr.handle[i])), range(len(self.contract_subscription_mgr.handle)))
+        
+        print subm
+        if subm:
+            self.tws_event_handler.broadcast_event('gw_subscriptions',  {'subscriptions': subm}, source='GW')
+        
+class SubscriptionManager():
+    
+    parent = None
+    # array list of contracts
+    handle = []
+    # contract key map to contract ID (index of the handle array)
+    tickerId = {}
+    
+    persist_f = None
+    
+    def __init__(self, parent=None):
+        self.parent = parent
+    
+   
+    def load_subscription(self, contracts):
+        for c in contracts:
+            self.reqMktData(c)
+            
+        self.dump()
+    
+    # returns -1 if not found, else the key id (which could be a zero value)
+    def is_subscribed(self, contract):
+        #print self.conId.keys()
+        ckey = ContractHelper.makeRedisKeyEx(contract) 
+        if ckey not in self.tickerId.keys():
+            return -1
+        else:
+            # note that 0 can be a key 
+            # be careful when checking the return values
+            # check for true false instead of using implicit comparsion
+            return self.tickerId[ckey]
+    
+
+#     def reqMktDataxx(self, contract):
+#         print '---------------'
+#         contractTuple = ('USO', 'STK', 'SMART', 'USD', '', 0.0, '')
+#         stkContract = self.makeStkContract(contractTuple)     
+#         stkContract.m_includeExpired = False       
+#         self.parent.connection.reqMktData(1, stkContract, '', False)     
+# 
+#         contractTuple = ('IBM', 'STK', 'SMART', 'USD', '', 0.0, '')
+#         stkContract = self.makeStkContract(contractTuple)
+#         stkContract.m_includeExpired = False
+#         print stkContract   
+#         print stkContract.__dict__         
+#         self.parent.connection.reqMktData(2, stkContract, '', False)     
+#             
+
+            
+    def reqMktData(self, contract):
+                  
+        
+        #logging.info('SubscriptionManager: reqMktData')
+  
+        def add_subscription(contract):
+            self.handle.append(contract)
+            newId = len(self.handle) - 1
+            self.tickerId[ContractHelper.makeRedisKeyEx(contract)] = newId 
+             
+            return newId
+  
+        id = self.is_subscribed(contract)
+        if id == -1: # not found
+            id = add_subscription(contract)
+
+            #
+            # the conId must be set to zero when calling TWS reqMktData
+            # otherwise TWS will fail to subscribe the contract
+            self.parent.connection.reqMktData(id, contract, '', False) 
+            
+                   
+#             if self.persist_f:
+#                 logging.debug('SubscriptionManager reqMktData: trigger callback')
+#                 self.persist_f(self.handle)
+                
+            logging.info('SubscriptionManager: reqMktData. Requesting market data, id = %d, contract = %s' % (id, ContractHelper.makeRedisKeyEx(contract)))
+        
+        else:    
+            self.parent.connection.reqMktData(1000 + id, contract, '', True)
+            logging.info('SubscriptionManager: reqMktData: contract already subscribed. Request snapshot = %d, contract = %s' % (id, ContractHelper.makeRedisKeyEx(contract)))
+        #self.dump()
+        
+#     def makeStkContract(self, contractTuple):
+#         newContract = Contract()
+#         newContract.m_symbol = contractTuple[0]
+#         newContract.m_secType = contractTuple[1]
+#         newContract.m_exchange = contractTuple[2]
+#         newContract.m_currency = contractTuple[3]
+#         newContract.m_expiry = contractTuple[4]
+#         newContract.m_strike = contractTuple[5]
+#         newContract.m_right = contractTuple[6]
+#         print 'Contract Values:%s,%s,%s,%s,%s,%s,%s:' % contractTuple
+#         return newContract        
+   
+    # use only after a broken connection is restored
+    # to re request market data 
+    def force_resubscription(self):
+        # starting from index 1 of the contract list, call  reqmktdata, and format the result into a list of tuples
+        for i in range(1, len(self.handle)):
+            self.parent.connection.reqMktData(i, self.handle[i], '', False)
+            logging.info('force_resubscription: %s' % ContractHelper.printContract(self.handle[i]))
+       
+            
+    def itemAt(self, id):
+        if id > 0 and id < len(self.handle):
+            return self.handle[id]
+        return -1
+
+    def dump(self):
+        
+        logging.info('subscription manager table:---------------------')
+        logging.info(''.join('%d: {%s},\n' % (i,  ''.join('%s:%s, ' % (k, v) for k, v in self.handle[i].__dict__.iteritems() )\
+                                     if self.handle[i] <> None else ''          ) for i in range(len(self.handle)))\
+                     )
+        
+        logging.info( ''.join('%s[%d],\n' % (k, v) for k, v in self.conId.iteritems()))
+        logging.info( 'Number of instruments subscribed: %d' % len(self.handle))
+        logging.info( '------------------------------------------------')
+        
+    def register_persistence_callback(self, func):
+        logging.info('subscription manager: registering callback')
+        self.persist_f = func
+        
+
+    
+        
+
+
+
+                
+    
+def test_subscription():        
+    s = SubscriptionManager()
+    contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20151029', 0, '')
+    c = ContractHelper.makeContract(contractTuple)   
+    print s.is_subscribed(c)
+    print s.add_subscription(c)
+    print s.is_subscribed(c)
+    s.dump()
+    
+    fr = open('/home/larry-13.04/workspace/finopt/data/subscription-hsio.txt')
+    for l in fr.readlines():
+        if l[0] <> '#':
+             
+            s.add_subscription(ContractHelper.makeContract(tuple([t for t in l.strip('\n').split(',')])))    
+    fr.close()
+    s.dump()
+    
+    fr = open('/home/larry-13.04/workspace/finopt/data/subscription-hsio.txt')
+    for l in fr.readlines():
+        if l[0] <> '#':
+             
+            print s.add_subscription(ContractHelper.makeContract(tuple([t for t in l.strip('\n').split(',')])))    
+    s.dump()
+    contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20151127', 0, '')
+    c = ContractHelper.makeContract(contractTuple)   
+    print s.is_subscribed(c)
+    print s.add_subscription(c)
+    print s.is_subscribed(c), ContractHelper.printContract(s.itemAt(s.is_subscribed(c))) 
+    
+    print 'test itemAt:'
+    contractTuple = ('HSI', 'OPT', 'HKFE', 'HKD', '20151127', 21400, 'C')
+    c = ContractHelper.makeContract(contractTuple)
+    print s.is_subscribed(c), ContractHelper.printContract(s.itemAt(s.is_subscribed(c)))
+    
+    
+
+
+    
+    
+if __name__ == '__main__':
+    
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0: 
+        raise ValueError, "Failed to open config file" 
+    
+   
+      
+    logconfig = eval(config.get("tws_gateway", "tws_gateway.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)        
+    
+
+    
+    
+    khost = config.get("epc", "kafka.host").strip('"').strip("'")
+    kport = config.get("epc", "kafka.port")
+    ihost = config.get("market", "ib.gateway").strip('"').strip("'")
+    iport = int(config.get("market", "ib.port"))
+    iappid = int(config.get("market", "ib.appid.portfolio"))   
+    
+    
+    
+    
+    #print 'give kafka server some time to register the topics...'
+    #sleep(2)
+    
+    app = TWS_gateway(ihost, iport, iappid, khost, kport, config)
+    app.start()
+     
+    print 'TWS_gateway started.'
+#     
+
+
+    
+
+
+#    test_subscription()
+    

comms/tws_protocol_helper.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import json
+
+class TWS_Protocol:
+    topicMethods = ('eConnect', 'reqAccountUpdates', 'reqOpenOrders', 'reqExecutions', 'reqIds', 'reqNewsBulletins', 'cancelNewsBulletins', \
+                      'setServerLogLevel', 'reqAccountSummary', 'reqPositions', 'reqAutoOpenOrders', 'reqAllOpenOrders', 'reqManagedAccts',\
+                       'requestFA', 'reqMktData', 'reqHistoricalData', 'placeOrder', 'eDisconnect')
+
+    topicEvents = ('accountDownloadEnd', 'execDetailsEnd', 'updateAccountTime', 'deltaNeutralValidation', 'orderStatus',\
+              'updateAccountValue', 'historicalData', 'openOrderEnd', 'updatePortfolio', 'managedAccounts', 'contractDetailsEnd',\
+              'positionEnd', 'bondContractDetails', 'accountSummary', 'updateNewsBulletin', 'scannerParameters', \
+              'tickString', 'accountSummaryEnd', 'scannerDataEnd', 'commissionReport', 'error', 'tickGeneric', 'tickPrice', \
+              'nextValidId', 'openOrder', 'realtimeBar', 'contractDetails', 'execDetails', 'tickOptionComputation', \
+              'updateMktDepth', 'scannerData', 'currentTime', 'error_0', 'error_1', 'tickSnapshotEnd', 'tickSize', \
+              'receiveFA', 'connectionClosed', 'position', 'updateMktDepthL2', 'fundamentalData', 'tickEFP')
+    
+    
+    
+    gatewayMethods = ('gw_req_subscriptions',)
+    gatewayEvents = ('gw_subscriptions',)
+    
+    oceMethods = ()
+    oceEvents = ('optionAnalytics',)
+    
+    
+    
+    def json_loads_ascii(self, data):    
+    
+        def ascii_encode_dict(data):
+            ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
+            return dict(map(ascii_encode, pair) for pair in data.items())    
+        
+        return json.loads(data, object_hook=ascii_encode_dict)
+    
+    
+    
+    
+class Message(object):
+    """ This class is taken out from IBpy 
+    everything is the same except it doesn't use __slots__
+    and thus allow setting variable values
+
+    """
+
+
+    def __init__(self, **kwds):
+        """ Constructor.
+
+        @param **kwds keywords and values for instance
+        """
+        
+        
+        for name in kwds.keys():
+            setattr(self, name, kwds[name])
+        
+        
+        #assert not kwds
+
+    def __len__(self):
+        """ x.__len__() <==> len(x)
+
+        """
+        return len(self.keys())
+
+    def __str__(self):
+        """ x.__str__() <==> str(x)
+
+        """
+        name = self.typeName
+        items = str.join(', ', ['%s=%s' % item for item in self.items()])
+        return '<%s%s>' % (name, (' ' + items) if items else '')
+
+    def items(self):
+        """ List of message (slot, slot value) pairs, as 2-tuples.
+
+        @return list of 2-tuples, each slot (name, value)
+        """
+        return zip(self.keys(), self.values())
+
+    def values(self):
+        """ List of instance slot values.
+
+        @return list of each slot value
+        """
+        return [getattr(self, key, None) for key in self.keys()]
+
+    def keys(self):
+        """ List of instance slots.
+
+        @return list of each slot.
+        """
+        return self.__dict__
+    
+    
+    
+def dummy():
+    topicsEvents = ['accountDownloadEnd', 'execDetailsEnd', 'updateAccountTime', 'deltaNeutralValidation', 'orderStatus',\
+          'updateAccountValue', 'historicalData', 'openOrderEnd', 'updatePortfolio', 'managedAccounts', 'contractDetailsEnd',\
+          'positionEnd', 'bondContractDetails', 'accountSummary', 'updateNewsBulletin', 'scannerParameters', \
+          'tickString', 'accountSummaryEnd', 'scannerDataEnd', 'commissionReport', 'error', 'tickGeneric', 'tickPrice', \
+          'nextValidId', 'openOrder', 'realtimeBar', 'contractDetails', 'execDetails', 'tickOptionComputation', \
+          'updateMktDepth', 'scannerData', 'currentTime', 'error_0', 'error_1', 'tickSnapshotEnd', 'tickSize', \
+          'receiveFA', 'connectionClosed', 'position', 'updateMktDepthL2', 'fundamentalData', 'tickEFP']
+
+    #s = ''.join('\tdef %s(self, items):\n\t\tself.handlemessage("%s", items)\n\n' % (s,s) for s in topicsEvents)
+    s = ''.join('# override this function to perform your own processing\n#\tdef %s(self, items):\n#\t\tpass\n\n' % s for s in topicsEvents)
+    return s
+    
+

config/app.cfg

@@ -36,6 +36,11 @@ tools.staticdir.tmpl : './html'
 tools.staticdir.on: True
 tools.staticdir.dir : './html/public'
 
+[/ws]
+tools.websocket.on: True
+tools.websocket.handler_cls: opt_serve.OptWebSocket
+
+
 
 [options_data]
 options_data.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/opt.log', 'filemode': 'w','level': logging.DEBUG}"
@@ -72,7 +77,10 @@ option.underlying = "('HSI', 'FUT', 'HKFE', 'HKD', '', 0, '')"
 option.underlying.month_price = "[['20151029', 22817.0, '^HSI'], ['20151127', 22715.0, '^HSI']]"
 option.underlying.yahoo_ws = "{'use_yahoo': True, 'func': 'ystockquote.get_price'}"
 option.underlying.tick_size = 200
-option.greeks.recal = "{'use_last_if_no_bidask': True, 'rate':0.005, 'div':0.005, 'vol':0.2}"
+
+# refer to this link for rate and div
+# https://www.hkex.com.hk/eng/sorc/tools/calculator_index_option.aspx
+option.greeks.recal = "{'use_last_if_no_bidask': True, 'rate':0.0009, 'div':0.328, 'vol':0.2}"
 option.chain_range = 0.08
 option.bid_ask_spread_tolerance = 0.90
 
@@ -106,11 +114,22 @@ ib_heartbeat.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/ib
 #ib_heartbeat.ib_port: 4001
 ib_heartbeat.ib_port: 7496
 ib_heartbeat.appid.id: 9911
-ib_heartbeat.gateway: 'localhost'
-#ib_heartbeat.gateway: '192.168.1.118'
-ib_heartbeat.try_interval: 3
-ib_heartbeat.suppress_msg_interval: 60
+#ib_heartbeat.gateway: 'localhost'
+ib_heartbeat.gateway: '192.168.1.118'
+ib_heartbeat.try_interval: 60
+ib_heartbeat.suppress_msg_interval: 120
 
 [smart_order]
 smart_order.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/smart_order.log', 'filemode': 'w','level': logging.INFO}"
 
+
+[tws_gateway]
+subscription_manager.subscriptions.redis_key: 'subscriptions'  
+tws_gateway.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/tws_gateway.log', 'filemode': 'w','level': logging.INFO}"
+tws_gateway.order_transmit: False
+
+
+[options_chain]
+options_calculation_engine.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/oce.log', 'filemode': 'w','level': logging.INFO}"
+option_chain_id.redis_key_prefix: 'optchain-'
+clear_redis_on_start: True

finopt/opt_serve.py

@@ -4,7 +4,7 @@ import logging
 import os
 import ast
 import urllib, urllib2, cookielib
-import datetime
+import datetime, time
 import re
 import json
 import cherrypy
@@ -16,6 +16,10 @@ import optcal
 import ConfigParser
 import portfolio
 from comms.alert_bot import AlertHelper
+from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
+from ws4py.websocket import WebSocket
+from ws4py.websocket import EchoWebSocket
+import thread
 
 
 class QServer(object):
@@ -149,9 +153,9 @@ class QServer(object):
                                 
             s = s + l + '],\n'
         
-        
+        print 'sorted months' + sorted_months[0]
         html_tmpl = html_tmpl.replace('{{{dataPremium}}}', s)
-
+        html_tmpl = html_tmpl.replace('{{{thisContractMonth}}}', sorted_months[0])
         
 
         
@@ -226,7 +230,8 @@ class QServer(object):
         
         
         html_tmpl = html_tmpl.replace('{{{dataPremium}}}', s)
-
+        
+        html_tmpl = html_tmpl.replace('{{{thisContractMonth}}}', sorted_months[0])
         
 
         
@@ -454,7 +459,7 @@ class QServer(object):
         
         colnames = "[['contract', 'strike', 'unreal PL', 'theta', 'delta'],"
         print '----------------------'
-        s_data = colnames + ''.join('["%s",%s,%s,%s,%s],' % (lcontract[i], lstrike[i], lupl[i], ltheta[i], lpos_delta[i]) for i in range(len(lcontract)))+ ']'
+        s_data = colnames + ''.join('["%s",%s,%s,%s,%s],' % (lcontract[i], lstrike[i], lupl[i], ltheta[i], abs(lpos_delta[i])) for i in range(len(lcontract)))+ ']'
       
         return s_data
     
@@ -481,9 +486,73 @@ class QServer(object):
         a = AlertHelper(self.config)
         a.post_msg(msg)  
         
+
+    @cherrypy.expose
+    def ws(self):
+        logging.info('at ws')
+        # you can access the class instance through
+        handler = cherrypy.request.ws_handler
+    
+        while handler.opened == False:
+            logging.info( 'not opened')
+        
+        logging.info( 'opened')
+        
+        
+    @cherrypy.expose
+    def ws_entry(self):
+        html = '%s%s/wstest.html' % (cherrypy.request.app.config['/']['tools.staticdir.root'], cherrypy.request.app.config['/static']['tools.staticdir.tmpl'])
+        f = open(html)
+        return f.read()
+    
+class OptWebSocket(WebSocket):
+    
+#     def __init__(self):
+#         logging.debug('instantiated.')
+
+
         
+    def received_message(self, message):
+        self.send(message.data, message.is_binary)
+        logging.info('received %s' % message.data)   
+        
+        
+#     def opened(self):
+#         logging.info('web socket opened')
+        #self.send('hello')
+#         while 1:
+#             self.send('%f' % time.time(), False)
+#             time.sleep(2)
+
+            
+
+    def opened(self):
+
+        logging.info('web socket opened')
+        def data_provider():   
+            
+            while 1:         
+#                print ('%f' % time.time())
+#                time.sleep(2)
             
-         
+                def cb():
+                    #for i in range(1, 200, 25):
+                    #    yield "#" * i
+                    yield '%f' % time.time()
+                
+                   
+                self.send(cb())
+        
+                logging.info('--- here')
+                time.sleep(2)  
+            
+        thread.start_new_thread(data_provider())
+        
+              
+    def closed(self, code, reason=None):
+        print "Closed down", code, reason
+        
+                 
 if __name__ == '__main__':
             
 #     logging.basicConfig(filename = "log/opt.log", filemode = 'a', 
@@ -515,7 +584,11 @@ if __name__ == '__main__':
     port = config.get("redis", "redis.port")
     db = config.get("redis", "redis.db")    
     r_conn = redis.Redis(host,port,db)
+
+
     
+    WebSocketPlugin(cherrypy.engine).subscribe()
+    cherrypy.tools.websocket = WebSocketTool()    
     cherrypy.quickstart(QServer(r_conn, config), '/', cfg_path[0])
     
    

finopt/optcal.py

@@ -111,35 +111,35 @@ if __name__ == '__main__':
     #spot 24119.0, X 25000, right: P, evaldate: 20150812, expiry: 20150828, rate: 0.0005, div: 0.0005, vol: 0.2000, premium: 334.0000
     #spot 24149.0, X 25200, right: P, evaldate: 20150812, expiry: 20150828, rate: 0.0005, div: 0.0005, vol: 0.2000, premium: 437.5000
     
-    results = cal_option(23067.0, 22000, 'P', '20151018', '20151030', 0.0009, 0.0328, 0.2918)
-    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
-    results = cal_option(23067.0, 22000, 'P', '20151018', '20151029', 0.0009, 0.0328, 0.2918)
-    npv1 = results['npv']
-    v1 = 0.2918
-
-    
-    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
-    results = cal_option(23067.0, 22000, 'C', '20151018', '20151029', 0.0009, 0.0328, 0.2918)
-    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
-
-    results = cal_option(23067.0, 22000, 'P', '20151018', '20151029', 0.0009, 0.0328, 0.2918*1.01)
-    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
-    npv2 = results['npv']
-    v2 = v1 * 1.01
-    
-    print 'validating vega: (%0.2f - %0.2f) / (%0.4f - %0.4f) = %0.2f' % (npv2, npv1, v2, v1, (npv2-npv1)/ (v2 - v1))     
-    print 'validating gamma: (%0.2f - %0.2f) / (%0.4f - %0.4f) = %0.2f' % (npv2, npv1, v2, v1, (npv2-npv1)/ (v2 - v1))
-    
-    results = cal_option(23067.0, 22000, 'C', '20151018', '20151029', 0.0009, 0.0328, 0.2918*1.01)
+    results = cal_option(22363.0, 22000, 'C', '20151201', '20151230', 0.00012, 0.0328, 0.198)
     print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+#     results = cal_option(23067.0, 22000, 'P', '20151018', '20151029', 0.0009, 0.0328, 0.2918)
+#     npv1 = results['npv']
+#     v1 = 0.2918
+# 
+#     
+#     print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+#     results = cal_option(23067.0, 22000, 'C', '20151018', '20151029', 0.0009, 0.0328, 0.2918)
+#     print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+# 
+#     results = cal_option(23067.0, 22000, 'P', '20151018', '20151029', 0.0009, 0.0328, 0.2918*1.01)
+#     print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+#     npv2 = results['npv']
+#     v2 = v1 * 1.01
+#     
+#     print 'validating vega: (%0.2f - %0.2f) / (%0.4f - %0.4f) = %0.2f' % (npv2, npv1, v2, v1, (npv2-npv1)/ (v2 - v1))     
+#     print 'validating gamma: (%0.2f - %0.2f) / (%0.4f - %0.4f) = %0.2f' % (npv2, npv1, v2, v1, (npv2-npv1)/ (v2 - v1))
+#     
+#     results = cal_option(23067.0, 22000, 'C', '20151018', '20151029', 0.0009, 0.0328, 0.2918*1.01)
+#     print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
 
     
  
     
-    results = cal_implvol(23067.0, 21000, 'P', '20151018', '20151030', 0.0009, 0.0328, 0.19, 14.5817)
+    #results = cal_implvol(22363.0, 22000, 'C', '20151201', '20151230', 0.0328, 0.00012, 0.21055, 
     
         
-    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+    #print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
 
 
     chk = HongKong()

finopt/options_analytics.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+import json
+import logging
+import ConfigParser
+from time import sleep
+import time, datetime
+from threading import Lock
+from kafka.client import KafkaClient
+from kafka import KafkaConsumer
+from kafka.producer import SimpleProducer
+from kafka.common import LeaderNotAvailableError, ConsumerTimeout
+import threading
+
+from misc2.helpers import ContractHelper
+import uuid
+
+
+from comms.tws_protocol_helper import TWS_Protocol
+
+class AnalyticsListener(threading.Thread):
+    consumer = None
+    command_handler = None
+    stop_consumer = False
+    
+    reg_all_callbacks = []
+
+
+
+    def __init__(self, host, port, id=None):
+
+        super(AnalyticsListener, self).__init__()
+        client = KafkaClient('%s:%s' % (host, port))
+        self.producer = SimpleProducer(client, async=False)
+        
+
+ 
+        # consumer_timeout_ms must be set - this allows the consumer an interval to exit from its blocking loop
+        self.consumer = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.oceEvents)] , \
+                                       metadata_broker_list=['%s:%s' % (host, port)],\
+                                       client_id = str(uuid.uuid1()) if id == None else id,\
+                                       group_id = 'epc.group',\
+                                       auto_commit_enable=True,\
+                                       consumer_timeout_ms = 2000,\
+                                       auto_commit_interval_ms=30 * 1000,\
+                                       auto_offset_reset='largest') # discard old ones
+        
+        self.reset_message_offset()
+        
+        
+        
+        
+
+    def reset_message_offset(self):
+        # 90 is a magic number or don't care (max_num_offsets)
+        topic_offsets =  map(lambda topic: (topic, self.consumer.get_partition_offsets(topic, 0, -1, 90)), TWS_Protocol.oceEvents)
+        topic_offsets = filter(lambda x: x <> None, map(lambda x: (x[0], x[1][1], max(x[1][0], 0)) if len(x[1]) > 1 else None, topic_offsets))
+        logging.info("AnalyticsListener: topic offset dump ------:")
+        logging.info (topic_offsets)
+        logging.info('AnalyticsListener set topic offset to the latest point\n%s' % (''.join('%s,%s,%s\n' % (x[0], x[1], x[2]) for x in topic_offsets)))
+        
+        # the set_topic_partitions call clears out all previous settings when starts
+        # therefore it's not possible to do something like this:
+        # self.consumer.set_topic_partitions(('gw_subscriptions', 0, 114,)
+        # self.consumer.set_topic_partitions(('tickPrice', 0, 27270,))
+        # as the second call will wipe out whatever was done previously
+        self.consumer.set_topic_partitions(*topic_offsets)        
+        
+
+                           
+    def run(self):
+
+ 
+            # keep running until someone tells us to stop
+            while self.stop_consumer == False:
+            #while True:
+  
+                    try:
+                        # the next() function runs an infinite blocking loop
+                        # it will raise a consumertimeout if no message is received after a pre-set interval   
+                        message = self.consumer.next()
+                        
+                        
+                        logging.debug("TWS_client_base_app: %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+                                                     message.offset, message.key,
+                                                     message.value))
+                        
+                        [f(message.value) for f in self.reg_all_callbacks]                        
+                        
+                        
+                    except ConsumerTimeout:
+                        logging.info('AnalyticsListener run: ConsumerTimeout. Check new message in the next round...')
+
+                                        
+    def stop(self):
+        logging.info('TWS_client_base_app: --------------- stopping consumer')
+        self.stop_consumer = True
+        
+
+    def registerAll(self, funcs):
+        [self.reg_all_callbacks.append(f) for f in funcs]
+        
+                

finopt/options_chain.py

@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import ConfigParser
+import json
+import logging
+import threading
+from ib.ext.Contract import Contract
+from misc2.helpers import ContractHelper, dict2str
+#from misc2.helpers import ContractHelper, OrderHelper, ExecutionFilterHelper
+from comms.tws_client import SimpleTWSClient
+from time import sleep
+import time, datetime
+import optcal
+import traceback
+import redis
+
+class Symbol():
+    
+    def __init__(self, contract):
+        self.contract = contract
+        self.tick_values = {}
+        self.extra = {}    
+    
+    def set_tick_value(self, id, price):
+        self.tick_values[id] = price
+
+    def get_tick_value(self, id):
+        try:
+            
+            return self.tick_values[id]
+    
+        except:
+            
+            return None
+
+    def get_contract(self):
+        return self.contract
+    
+    def get_tick_values(self):
+        return self.tick_values
+    
+    def set_extra_attributes(self, id, val):
+        self.extra[id] = val
+            
+    def get_extra_attributes(self):
+        return self.extra
+    
+class Option(Symbol):
+    
+    analytics = None
+    IMPL_VOL = 'imvol'
+    DELTA    = 'delta'
+    GAMMA    = 'gamma'
+    THETA    = 'theta'
+    VEGA     = 'vega'
+    PREMIUM  = 'npv'
+    
+    
+    #[0,1,2,3,4,5,6,7,8,9,14,5001,5002,5003,5004,5005,5006]
+        
+    def __init__(self, contract):
+        Symbol.__init__(self, contract)
+        
+        self.set_analytics(-1.0, -1.0, -1.0, -1.0, -1.0, -1.0)
+
+        
+    def set_analytics(self, imvol=None, delta=None, gamma=None, theta=None, vega=None, npv=None):
+        
+        
+        if self.analytics == None:
+            self.analytics = {}           
+        self.analytics[Option.IMPL_VOL] = imvol
+        self.analytics[Option.DELTA] = delta 
+        self.analytics[Option.GAMMA] = gamma
+        self.analytics[Option.THETA] = theta
+        self.analytics[Option.VEGA] = vega
+        self.analytics[Option.PREMIUM] = npv
+        
+        
+    def get_analytics(self):
+        return self.analytics
+    
+    
+    def object2kvstring(self):
+        
+        try:           
+            kv = self.object2kv()
+            return '{"%s":%s, "%s":%s, "%s":%s, "%s":%s}' % ('analytics', dict2str(kv['analytics']), 
+                                                    'contract', ContractHelper.contract2kvstring(self.get_contract()), 
+                                                    'tick_values', dict2str(kv['tick_values']),
+                                                    'extra', dict2str(kv['extra']))
+        except:
+            logging.error( 'Exception Option.object2kvstring')
+               
+        return None
+    
+    
+    def object2kv(self):
+        try:
+            analytics = self.get_analytics()
+            contract =  self.get_contract()
+            tick_values = self.get_tick_values()
+            extra = self.get_extra_attributes()
+            return {'analytics': analytics, 'contract': contract, 'tick_values': tick_values, 'extra': extra}            
+        except:
+            logging.error( 'Exception Option.object2kv')
+               
+        return None
+        
+            
+    
+               
+        
+
+class OptionsChain():
+    underlying = None
+    spd_size = None
+    multiplier = None
+    
+    #
+    # options is a list containing Option object
+    options = None
+    
+    id = None
+    div = 0.0
+    rate = 0.0
+    expiry = None
+    trade_vol = None
+    #iv = optcal.cal_implvol(spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate, div, vol, premium)
+    
+    
+    def __init__(self, id):
+        self.id = id
+        self.options = []
+        
+    
+    def get_id(self):
+        return self.id
+    
+    def get_underlying(self):
+        return self.underlying
+    
+    def set_underlying(self, contract):
+        #self.underlying = contract
+        self.underlying = Symbol(contract)
+        
+    def set_spread_table(self, spd_size, multiplier):
+        self.spd_size = spd_size
+        self.multiplier = multiplier
+    
+    def set_div(self, div):
+        self.div = div
+    
+    def set_rate(self, rate):
+        self.rate = rate
+        
+    def set_expiry(self, expiry):
+        self.expiry = expiry
+    
+    def set_trade_vol(self, tvol):
+        self.trade_vol = tvol
+        
+    def set_option_structure(self, underlying, spd_size, multiplier, rate, div, expiry):
+        self.set_div(div)
+        self.set_rate(rate)
+        self.set_spread_table(spd_size, multiplier)
+        self.set_underlying(underlying)
+        self.set_expiry(expiry)
+      
+    def build_chain(self, px, bound, trade_vol):
+        self.set_trade_vol(trade_vol)
+        undlypx = round(px  / self.spd_size) * self.spd_size
+        upper_limit = undlypx * (1 + bound)
+        lower_limit = undlypx * (1 - bound)          
+        
+        base_opt_contract = json.loads(ContractHelper.object2kvstring(self.get_underlying().get_contract()))
+
+        #for i in self.xfrange(int(undlypx), int(upper_limit ), self.spd_size):
+        for i in self.xfrange(undlypx, upper_limit, self.spd_size):
+
+            base_opt_contract['m_secType'] = 'OPT'
+            base_opt_contract['m_strike'] = i
+            base_opt_contract['m_expiry'] = self.expiry
+            base_opt_contract['m_right'] = 'C'
+            base_opt_contract['m_multiplier'] = self.multiplier
+            #self.options.append(ContractHelper.kv2object(base_opt_contract, Contract))
+            self.options.append(Option(ContractHelper.kv2object(base_opt_contract, Contract)))
+            
+            base_opt_contract['m_right'] = 'P'
+            #self.options.append(ContractHelper.kv2object(base_opt_contract, Contract))
+            self.options.append(Option(ContractHelper.kv2object(base_opt_contract, Contract)))
+ 
+        for i in self.xfrange(undlypx - self.spd_size, lower_limit, -self.spd_size):             
+            base_opt_contract['m_secType'] = 'OPT'
+            base_opt_contract['m_strike'] = i
+            base_opt_contract['m_expiry'] = self.expiry
+            base_opt_contract['m_right'] = 'C'
+            base_opt_contract['m_multiplier'] = self.multiplier
+            #self.options.append(ContractHelper.kv2object(base_opt_contract, Contract))
+            self.options.append(Option(ContractHelper.kv2object(base_opt_contract, Contract)))
+             
+            base_opt_contract['m_right'] = 'P'
+            #self.options.append(ContractHelper.kv2object(base_opt_contract, Contract))
+            self.options.append(Option(ContractHelper.kv2object(base_opt_contract, Contract)))
+        
+#         print '------------####'     
+#         for c in self.options:
+#              #print ContractHelper.contract2kvstring(c)
+#              print c.option2kv()
+        
+
+    def xfrange(self, start, stop=None, step=None):
+        if stop is None:
+            stop = float(start)
+            start = 0.0
+        
+        if step is None:
+            step = 1.0
+        
+        cur = float(start)
+        if start <= stop:
+            while cur < stop:
+                yield cur
+                cur += step
+        else:
+            while cur > stop:
+                yield cur
+                cur += step
+            
+    def get_option_chain(self):
+        return self.options
+
+        
+    def add_option(self, kvc):
+        pass
+    
+    
+    def pretty_print(self):
+        sorted_opt = sorted(map(lambda i: (self.options[i].get_contract().m_strike, self.options[i]) , range(len(self.options))))
+        
+        def format_tick_val(val, fmt):
+            if val == None:
+                length = len(fmt % (0))
+                return ' ' * length
+            
+            return fmt % (val) 
+        
+
+
+        
+        sorted_call = filter(lambda x: x[1].get_contract().m_right == 'C', sorted_opt)
+        sorted_put = filter(lambda x: x[1].get_contract().m_right == 'P', sorted_opt)
+        # last, bidq, bid, ask, askq, imvol, delta, theta
+        fmt_spec = '%8.2f'
+        fmt_spec2 = '%8.4f'
+        fmt_specq = '%8d'
+        fmt_call = map(lambda x: (x[0], '%s,%s,%s,%s,%s,%s,%s,%s' % (format_tick_val(x[1].get_tick_value(4), fmt_spec),
+                                               format_tick_val(x[1].get_tick_value(0), fmt_specq),
+                                               format_tick_val(x[1].get_tick_value(1), fmt_spec),
+                                               format_tick_val(x[1].get_tick_value(2), fmt_spec),
+                                               format_tick_val(x[1].get_tick_value(3), fmt_specq),
+                                               format_tick_val(x[1].get_analytics()[Option.IMPL_VOL], fmt_spec2),
+                                               format_tick_val(x[1].get_analytics()[Option.DELTA], fmt_spec2),
+                                               format_tick_val(x[1].get_analytics()[Option.THETA], fmt_spec2),
+                                               )), sorted_call)
+        
+        fmt_put = map(lambda x: (x[0], '%s,%s,%s,%s,%s,%s,%s,%s' % (format_tick_val(x[1].get_tick_value(4), fmt_spec),
+                                               format_tick_val(x[1].get_tick_value(0), fmt_specq),                                                                                                                  
+                                               format_tick_val(x[1].get_tick_value(1), fmt_spec),
+                                               format_tick_val(x[1].get_tick_value(2), fmt_spec),
+                                               format_tick_val(x[1].get_tick_value(3), fmt_specq),
+                                               format_tick_val(x[1].get_analytics()[Option.IMPL_VOL], fmt_spec2),
+                                               format_tick_val(x[1].get_analytics()[Option.DELTA], fmt_spec2),
+                                               format_tick_val(x[1].get_analytics()[Option.THETA], fmt_spec2),                    
+                                               )), sorted_put)
+        title = '%s%30s%s' % ('-' * 40, ContractHelper.makeRedisKeyEx(self.get_underlying().get_contract()).center(50, ' '), '-' * 40) 
+        header = '%8s|%8s|%8s|%8s|%8s|%8s|%8s|%8s |%8s| %8s|%8s|%8s|%8s|%8s|%8s|%8s|%8s' % ('last', 'bidq', 'bid', 'ask', 'askq', 'ivol', 'delta', 'theta', 'strike', 'last', 'bidq', 'bid', 'ask', 'askq', 'ivol', 'delta', 'theta')
+        combined = map(lambda i: '%s |%8.2f| %s' % (fmt_call[i][1], fmt_put[i][0], fmt_put[i][1]), range(len(fmt_call)) )
+        footer = '%s' % ('-' * 130) 
+        print title
+        print header
+        for e in combined:
+            print e
+        print footer
+        
+        
+
+                
+        
+    
+
+class OptionsCalculationEngine(SimpleTWSClient):
+    tickerMap = {}
+    option_chains = {}    
+    appid = 'OCE'
+    
+    download_gw_map_done = False
+    rs = None
+    rs_oc_prefix = None
+    
+    def __init__(self, config): #host, port, id=None):
+        
+        
+        khost = config.get("epc", "kafka.host").strip('"').strip("'")
+        kport = config.get("epc", "kafka.port")
+        
+             
+        super(OptionsCalculationEngine, self).__init__(khost, kport, self.appid)
+        logging.info('OptionsCalculationEngine client id=%s' % id)    
+        
+        self.initialize_redis(config)
+
+    def initialize_redis(self, config):
+        r_host = config.get("redis", "redis.server").strip('"').strip("'")
+        r_port = config.get("redis", "redis.port")
+        r_db = config.get("redis", "redis.db")     
+        clr_flag = config.get("options_chain", "clear_redis_on_start")
+        
+        self.rs = redis.Redis(r_host, r_port, r_db)
+        try:
+            if clr_flag.upper() == 'TRUE':
+                
+                logging.info('clear previously saved option chain values in redis...')
+                # clear previously saved option chain values
+                self.rs_oc_prefix = config.get("options_chain", "option_chain_id.redis_key_prefix").strip('"').strip("'")
+                # find all chains with keys beginning like rs_oc_prefix
+                oc_sets = self.rs.keys(pattern = '%s*' % self.rs_oc_prefix)
+                
+                for oc_set in oc_sets:
+                    l = map(lambda x: self.rs.srem(oc_set, x), list(self.rs.smembers(oc_set)))
+                    self.rs.delete(oc_set)
+                
+            
+            #self.rs.client_list()
+        except redis.ConnectionError:
+            logging.error('TWS_gateway: unable to connect to redis server using these settings: %s port:%d db:%d' % (r_host, r_port, r_db))
+            logging.error('aborting...')
+            sys.exit(-1)
+    
+    def add_chain(self, option_chain):
+        self.option_chains[option_chain.get_id()] = option_chain
+    
+
+
+        
+    def map_gw_ticker_to_option_chains(self):
+
+        for k, option_chain in self.option_chains.iteritems():
+            for option in option_chain.get_option_chain():
+                self.get_command_handler().reqMktData(option.get_contract())
+            self.get_command_handler().reqMktData(option_chain.get_underlying().get_contract())
+
+
+        
+        self.get_command_handler().gw_req_subscriptions()
+        while not self.download_gw_map_done:
+            sleep(1)
+            logging.info('map_gw_ticker_to_option_chains: awaiting subscription table down to get done...')
+            pass
+        
+        logging.info('map_gw_ticker_to_option_chains: complete download subscription table from gw')
+        
+        # for each tick id in the tickerMap, compare...
+        for tickerId, cv in self.tickerMap.iteritems():
+            
+            # for each chain in the option chain, iterate...
+            for chain_id, chain in self.option_chains.iteritems():
+                
+                # for each of the item (Option object) within the option chain...
+                for i in range(len(chain.get_option_chain())):
+                    
+                    # if a match is found...                 
+                    if cv['contract'] == chain.get_option_chain()[i].get_contract():
+                        print 'id:%s col:%d -> tickerId: %d >> %s %s' % (chain_id, i, tickerId, ContractHelper.makeRedisKeyEx(cv['contract']),\
+                                                                          ContractHelper.makeRedisKeyEx(chain.get_option_chain()[i].get_contract()))
+                        
+                        # update the ticker map
+                        # key= tws ticker id
+                        # value-> key: tick2oc_slot (ticker map to option chain slot) => [chain_id, the ith element]
+                        cv['tick2oc_slot'] = [chain_id, i]
+                        
+                if chain.get_underlying().get_contract() == cv['contract']:
+                        print 'id:%s col:%d -> tickerId: %d >> %s %s' % (chain_id, i, tickerId, ContractHelper.makeRedisKeyEx(cv['contract']),\
+                                                                          ContractHelper.makeRedisKeyEx(chain.get_underlying().get_contract()))
+                        cv['tick2oc_slot'] = [chain_id, -999]
+        
+
+    
+    def get_option_in_chain(self, chain_id, elem_at):
+        return self.option_chains[chain_id].get_option_chain()[elem_at]
+    
+    def get_underlying_in_chain(self, chain_id):
+        return self.option_chains[chain_id].get_underlying()
+    
+    def run_server(self):
+        self.connect()
+            
+        self.map_gw_ticker_to_option_chains()
+        
+        #sleep(5)
+        while 1:
+            sleep(5)
+            for oc in self.option_chains.keys():
+                self.option_chains[oc].pretty_print()
+            
+        self.disconnect()
+        
+        
+        
+######################################################
+#   TWS messages
+    def dump(self, msg_name, mapping):
+        # the mapping is a comms.tws_protocol_helper.Message object
+        # which can be accessed directly using the __dict__.['xxx'] method 
+        items = list(mapping.items())
+        items.sort()
+        print ('>>> %s <<< %s' % (msg_name, ''.join('%s=%s, '% (k, v if k <> 'ts' else datetime.datetime.fromtimestamp(v).strftime('%Y-%m-%d %H:%M:%S.%f')) for k, v in items))) 
+    
+        
+    # override the tickSize message
+    def tickSize(self, items):
+    
+        try:
+            contract = self.tickerMap[items.__dict__['tickerId']]['contract']
+            tick2oc_slot = self.tickerMap[items.__dict__['tickerId']]['tick2oc_slot']
+            field = items.__dict__['field']
+            
+            logging.debug('tickSize>> %s' % ('[%d:%s:%s:%d] %s=%d %0.2f [%s]' % \
+                                        (items.__dict__['tickerId'], ContractHelper.makeRedisKeyEx(contract),\
+                                         tick2oc_slot[0], tick2oc_slot[1],\
+                                        'bid_q' if field == 0 else ('ask_q' if field == 3 else ('last_q' if field == 5 else field)), \
+                                        items.__dict__['size'], self.option_chains[tick2oc_slot[0]].multiplier,\
+                                        datetime.datetime.fromtimestamp(items.__dict__['ts']).strftime('%Y-%m-%d %H:%M:%S.%f')))
+                          )
+            # is an option
+            if tick2oc_slot[1] <> -999:
+                o = self.get_option_in_chain(tick2oc_slot[0], tick2oc_slot[1])
+                o.set_tick_value(field, items.__dict__['size'])
+
+            # is an underylying  
+            else:             
+                
+                self.get_underlying_in_chain(tick2oc_slot[0]).set_tick_value(field, items.__dict__['size'])
+                #print 'set fut price %s %d:%0.2f' % (ContractHelper.makeRedisKeyEx(self.get_underlying_in_chain(tick2oc_slot[0]).get_contract()), field, items.__dict__['price'])
+            
+            
+        except KeyError:
+            logging.error('tickSize: keyerror: (this could happen on the 1st run as the subscription manager sub list is still empty.')
+            logging.error(''.join('%s=%s, '% (k,v) for k,v in items.__dict__.iteritems()))
+
+
+
+
+    def tickPrice(self, items):
+        try:
+            contract = self.tickerMap[items.__dict__['tickerId']]['contract']
+            field = items.__dict__['field']
+            tick2oc_slot = self.tickerMap[items.__dict__['tickerId']]['tick2oc_slot']
+            today = time.strftime('%Y%m%d')
+            price = items.__dict__['price']
+            
+            #
+            # perform some sanity check
+            # 
+            # if field is not bid, ask, last, or close, pass
+            if field not in [1,2,4,9]:
+                logging.debug('tickPrice: discard unwanted msg field:%d' % field)
+                return
+            
+            # if we received a negative price, pass
+            if price == -1:
+                logging.debug('tickPrice: discard unwanted msg price==-1')
+                return 
+        
+            logging.debug( 'tickPrice>> %s' % ('[%d:%s:%s:%d] %s=%0.4f [%s]' % \
+                                        (items.__dict__['tickerId'], ContractHelper.makeRedisKeyEx(contract),\
+                                         tick2oc_slot[0], tick2oc_slot[1],\
+                                        'bid' if field == 1 else ('ask' if field == 2 else ('last' if field == 4 else field)), \
+                                        items.__dict__['price'], datetime.datetime.fromtimestamp(items.__dict__['ts']).strftime('%Y-%m-%d %H:%M:%S.%f')))
+                          )
+        
+            # is an option
+            if tick2oc_slot[1] <> -999:
+                o = self.get_option_in_chain(tick2oc_slot[0], tick2oc_slot[1])
+                o.set_tick_value(field, items.__dict__['price'])
+
+                try:
+                    spot = self.get_underlying_in_chain(tick2oc_slot[0]).get_tick_value(4)
+                    
+                    # the underlying price may not be available when we receive tick price for options
+                    if spot <> None:
+                        
+                    
+                        rate = self.option_chains[tick2oc_slot[0]].rate
+                        div = self.option_chains[tick2oc_slot[0]].div
+                        tvol = self.option_chains[tick2oc_slot[0]].trade_vol
+                        logging.debug('sp=%0.4f, x=%0.4f, %s, evald=%s, expiryd=%s, r=%0.4f, d=%0.4f, v=%0.4f, px[%d]=%0.4f' % (\
+                                                spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate,\
+                                                div, tvol, field, items.__dict__['price']))
+                        
+                        results = None                  
+                        iv = optcal.cal_implvol(spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate,\
+                                                div, tvol, items.__dict__['price'])
+                        results = optcal.cal_option(spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate, div, iv['imvol'])
+                        results[Option.IMPL_VOL] = iv['imvol']
+                        #print results
+                        o.set_analytics(**results)
+                        logging.debug(o.get_analytics())
+                        o.set_extra_attributes('spot', spot)
+                        o.set_extra_attributes('rate', rate)
+                        o.set_extra_attributes('div', div)   
+                        o.set_extra_attributes('chain_id', tick2oc_slot[0])
+                      
+                                  
+                except Exception, err:
+                    logging.error(traceback.format_exc())
+
+                
+
+                o.set_extra_attributes('last_updated', datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
+                self.broadcast_analytics(tick2oc_slot[0], o)
+                logging.debug(o.object2kvstring())
+            # is an underylying  
+            else:             
+                
+                self.get_underlying_in_chain(tick2oc_slot[0]).set_tick_value(field, items.__dict__['price'])
+                #print 'set fut price %s %d:%0.2f' % (ContractHelper.makeRedisKeyEx(self.get_underlying_in_chain(tick2oc_slot[0]).get_contract()), field, items.__dict__['price'])
+            
+        except KeyError:
+            logging.error('tickPrice: keyerror: (this could happen on the 1st run as the subscription manager sub list is still empty.')
+            logging.error(''.join('%s=%s, '% (k,v) for k,v in items.__dict__.iteritems()))
+            
+            
+    def tickString(self, items):
+        pass
+
+    def tickGeneric(self, items):
+        pass
+
+    def tickSnapshotEnd(self, items):
+        pass
+
+    def error(self, items):
+        self.dump('error', items)
+
+    def error_0(self, items):
+        self.dump('error', items)
+ 
+    def error_1(self, items):
+        self.dump('error', items)
+
+
+
+######################################################
+#   GW messages
+          
+    def gw_subscriptions(self, items):
+        # <class 'comms.tws_protocol_helper.Message'>
+        # sample
+        #{0: {'contract': <ib.ext.Contract.Contract object at 0x7ff8f8c9e210>}, 1: {'contract': <ib.ext.Contract.Contract object at 0x7ff8f8c9e250>},... }
+        #print items.__dict__['subscriptions']
+        
+        l = map(lambda x: {x[0]: {'contract': x[1]}}, map(lambda x: (x[0], ContractHelper.kvstring2object(x[1], Contract)), items.__dict__['subscriptions']))
+        #l = map(lambda x: {x[0]: x[1]}, map(lambda x: (x[0], json.loads(x[1])), items.__dict__['subscriptions']))
+        for i in l:
+            self.tickerMap.update(i)   
+        logging.info('gw_subscriptions -> dump tickerMap ')
+        logging.info(''.join('%s=%s,' % (k,ContractHelper.makeRedisKeyEx(v['contract'])) for k,v in self.tickerMap.iteritems())) 
+    
+        self.download_gw_map_done = True
+
+    
+    
+    def broadcast_analytics(self, chain_id, option):
+        
+        o_key = ContractHelper.makeRedisKeyEx(option.get_contract())
+        self.rs.sadd('%s%s' % (self.rs_oc_prefix, chain_id), o_key)
+        
+        msg_str = option.object2kvstring()
+        self.rs.set(o_key, msg_str)
+        
+        
+        self.get_producer().send_messages('optionAnalytics', msg_str)
+    
+    
+
+        
+    
+if __name__ == '__main__':
+    
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0: 
+        raise ValueError, "Failed to open config file" 
+    
+   
+      
+    logconfig = eval(config.get("options_chain", "options_calculation_engine.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)        
+        
+    
+    contractTuple = ('QQQ', 'STK', 'SMART', 'USD', '', 0, '')
+    contract = ContractHelper.makeContract(contractTuple)  
+    oc = OptionsChain('QQQ-DEC11')
+    oc.set_option_structure(contract, 0.5, 100, 0.005, 0.003, '20151211')
+    oc.build_chain(114.79, 0.05, 0.25)
+    for c in oc.get_option_chain():
+        print '%s' % ContractHelper.makeRedisKeyEx(c.get_contract())
+    
+
+    contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20151230', 0, '')
+    contract = ContractHelper.makeContract(contractTuple)  
+    oc1 = OptionsChain('HSI-DEC30')
+    oc1.set_option_structure(contract, 200, 50, 0.0012, 0.0328, '20151230')
+    oc1.build_chain(22508, 0.08, 0.219)
+    for c in oc1.get_option_chain():
+        print '%s' % ContractHelper.makeRedisKeyEx(c.get_contract())
+
+
+    contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20160128', 0, '')
+    contract = ContractHelper.makeContract(contractTuple)  
+    oc2 = OptionsChain('HSI-JAN28')
+    oc2.set_option_structure(contract, 200, 50, 0.0012, 0.0328, '20160128')
+    oc2.build_chain(22508, 0.08, 0.22)
+    for c in oc2.get_option_chain():
+        print '%s' % ContractHelper.makeRedisKeyEx(c.get_contract())
+
+
+
+    
+    oce = OptionsCalculationEngine(config)
+#    oce.add_chain(oc)
+    oce.add_chain(oc1)
+    oce.add_chain(oc2)
+    oce.run_server()

finopt/options_data.py

@@ -6,7 +6,7 @@ import logging
 import thread
 import ConfigParser
 from ib.ext.Contract import Contract
-from ib.opt import ibConnection, message
+from ib.opt import ibConnection#, message
 from time import sleep
 import time, datetime
 import optcal
@@ -15,6 +15,8 @@ import cherrypy
 import redis
 import ystockquote
 import portfolio
+from misc2.helpers import ContractHelper
+
             
 # Tick Value      Description
 # 5001            impl vol
@@ -431,90 +433,6 @@ class MarketDataManager():
             logging.debug("cancelling tick id subscription %d" % i)
         self.con.disconnect()
 
-class ContractHelper():
-    
-    def __init__(self, contractTuple):
-        self.makeContract(contractTuple)
-    
-    @staticmethod
-    def makeContract(contractTuple):
-        newContract = Contract()
-        newContract.m_symbol = contractTuple[0]
-        newContract.m_secType = contractTuple[1]
-        newContract.m_exchange = contractTuple[2]
-        newContract.m_currency = contractTuple[3]
-        newContract.m_expiry = contractTuple[4]
-        newContract.m_strike = contractTuple[5]
-        newContract.m_right = contractTuple[6]
-        logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s:' % contractTuple)
-        return newContract
-    
-    @staticmethod
-    def convert2Tuple(newContract):
-        newContractTuple = (newContract.m_symbol,\
-                            newContract.m_secType,\
-                            newContract.m_exchange,\
-                            newContract.m_currency,\
-                            newContract.m_expiry,\
-                            newContract.m_strike,\
-                            newContract.m_right, newContract.m_conId)
-        logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s %s:' % newContractTuple)
-        return newContractTuple
-    
-    
-    @staticmethod
-    def printContract(contract):
-        s = '[%s-%s-%s-%s-%s-%s-%s-%s]' % (contract.m_symbol,
-                                                       contract.m_secType,
-                                                       contract.m_exchange,
-                                                       contract.m_currency,
-                                                       contract.m_expiry,
-                                                       contract.m_strike,
-                                                       contract.m_right,
-                                                       contract.m_conId)
-        #logging.info(s)
-        return s
-    
-    @staticmethod
-    def makeRedisKey(contract):
-        #print "makerediskey %s" % ContractHelper.printContract(contract)
-#20150904        
-        contract.m_strike = int(contract.m_strike)
-        
-        if contract.m_secType == 'OPT':
-            s = '%s-%s-%s-%s' % (contract.m_symbol,
-                                                           contract.m_expiry,
-                                                           contract.m_strike,
-                                                           contract.m_right)
-        else:
-            s = '%s-%s-%s-%s' % (contract.m_symbol,
-                                                           contract.m_expiry,
-                                                           contract.m_secType, '')
-            
-        return s
-      
-    @staticmethod
-    def makeRedisKeyEx(contract, old=False):
-        # this routine is to circumvent a problem in makeRedisKey with 
-        # the key in position 3 having different meanings under different conditions.
-        #  
-        # 
-        # to differentiate the keys generated by the old and new functions, 
-        # contract keys created using this routine have their last slot
-        # hard coded a magic number 102
-        
-        if (old):
-            return ContractHelper.makeRedisKey(contract)
-        
-        contract.m_strike = int(contract.m_strike)
-        s = '%s-%s-%s-%s-%s-%s-%d' % (contract.m_symbol,
-                                                           contract.m_expiry,
-                                                           contract.m_strike,
-                                                           contract.m_right,
-                                                           contract.m_secType,
-                                                           contract.m_currency,
-                                                           102)
-        return s
       
 def test():
 #     contractTuple = ('USD', 'CASH', 'IDEALPRO', 'JPY', '', 0.0, '')
@@ -709,6 +627,7 @@ class DataMap():
     def printMapCSV(self):
         keys = [0,1,2,3,4,5,6,7,8,9,14,5001,5002,5003,5004,5005,5006]
         print 'tick id,contract,' + ','.join(str(k) for k in keys)
+        lines = ''
         for tick_id, instr_v in DataMap().getData().iteritems():
             
             #s = '%s,%s-%s-%s,' % (tick_id, DataMap().get(tick_id)['contract'].m_right, DataMap().get(tick_id)['contract'].m_expiry, DataMap().get(tick_id)['contract'].m_strike)
@@ -718,8 +637,11 @@ class DataMap():
                     s = s + "%s, " % (instr_v[k])
                 else:
                     s = s + ','
-                 
-            print "%s" % s
+            
+            print s
+
+            lines += "%s\n" % s
+        return lines
         
         
     def createImplVolChartData(self):    
@@ -742,7 +664,10 @@ class DataMap():
 #                    matrix[strike][expiry][right] = ['null','null']
 #20150915                     
                     matrix[strike][expiry][right] = ['null','null','null','null']
+                logging.debug('-------------########################------------------- instr_v.keys')
+                logging.debug(instr_v.keys())
                 if 5001 in instr_v.keys(): 
+                    
                     matrix[strike][expiry][right][0] =  (instr_v[5001]) if self.isfloat(instr_v[5001]) else 'null'
                 # if bid and ask is available show mid price, else use last price
                 # but first verify that bid ask spread are not wider than 10%, else use last price
@@ -770,8 +695,8 @@ class DataMap():
 # [0] delta, [1] mid or last, [2]                                    
                     matrix[strike][expiry][right][2] =  (instr_v[1] if instr_v[1] > 0 else matrix[strike][expiry][right][1]) if self.isfloat(instr_v[1]) else 'null'
                     matrix[strike][expiry][right][3] =  (instr_v[2] if instr_v[2] > 0 else matrix[strike][expiry][right][1]) if self.isfloat(instr_v[2]) else 'null'
-                    logging.debug( 'createImplVolChartData: unusable bid/ask prices. bid and ask %s,%s' % \
-                                   (str(matrix[strike][expiry][right][2]), str(matrix[strike][expiry][right][2])))
+                    logging.debug( 'createImplVolChartData: bid/ask prices. bid and ask %s,%s' % \
+                                   (str(matrix[strike][expiry][right][2]), str(matrix[strike][expiry][right][3])))
                         
                 else:
                     if 4 in instr_v.keys():
@@ -891,18 +816,159 @@ class DataMap():
                     
             sleep(sec)
 
+
+
+
+# class ContractHelper():
+#       
+#     def __init__(self, contractTuple):
+#         self.makeContract(contractTuple)
+#       
+#       
+#     @staticmethod
+#     def makeContract(contractTuple):
+#         newContract = Contract()
+#         newContract.m_symbol = contractTuple[0]
+#         newContract.m_secType = contractTuple[1]
+#         newContract.m_exchange = contractTuple[2]
+#         newContract.m_currency = contractTuple[3]
+#         newContract.m_expiry = contractTuple[4]
+#         newContract.m_strike = contractTuple[5]
+#         newContract.m_right = contractTuple[6]
+#         logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s:' % contractTuple)
+#         return newContract
+#       
+#     @staticmethod
+#     def convert2Tuple(newContract):
+#         newContractTuple = (newContract.m_symbol,\
+#                             newContract.m_secType,\
+#                             newContract.m_exchange,\
+#                             newContract.m_currency,\
+#                             newContract.m_expiry,\
+#                             newContract.m_strike,\
+#                             newContract.m_right, newContract.m_conId)
+#         logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s %s:' % newContractTuple)
+#         return newContractTuple
+#       
+#   
+#     @staticmethod
+#     def contract2mapstring(contract):
+#         return json.dumps(contract.__dict__)
+#   
+#   
+#   
+#     @staticmethod
+#     def mapstring2contract(sm_contract):
+#           
+#         newContract = Contract()
+#         mapContract = json.loads(sm_contract)
+#         map(lambda x: newContract.__setattr__(x, mapContract[x].encode('ascii') if type(mapContract[x]) == unicode else mapContract[x]), mapContract.keys())
+#         return newContract
+#   
+#   
+#     @staticmethod
+#     def map2contract(m_contract):
+#           
+#         newContract = Contract()
+#         map(lambda x: newContract.__setattr__(x, m_contract[x].encode('ascii') if type(m_contract[x]) == unicode else m_contract[x]), m_contract.keys())
+#         return newContract
+#       
+#   
+#   
+#       
+#     @staticmethod
+#     def printContract(contract):
+#         s = '[%s-%s-%s-%s-%s-%s-%s-%s]' % (contract.m_symbol,
+#                                                        contract.m_secType,
+#                                                        contract.m_exchange,
+#                                                        contract.m_currency,
+#                                                        contract.m_expiry,
+#                                                        contract.m_strike,
+#                                                        contract.m_right,
+#                                                        contract.m_conId)
+#         #logging.info(s)
+#         return s
+#       
+#     @staticmethod
+#     def makeRedisKey(contract):
+#         #print "makerediskey %s" % ContractHelper.printContract(contract)
+# #20150904        
+#         contract.m_strike = int(contract.m_strike)
+#           
+#         if contract.m_secType == 'OPT':
+#             s = '%s-%s-%s-%s' % (contract.m_symbol,
+#                                                            contract.m_expiry,
+#                                                            contract.m_strike,
+#                                                            contract.m_right)
+#         else:
+#             s = '%s-%s-%s-%s' % (contract.m_symbol,
+#                                                            contract.m_expiry,
+#                                                            contract.m_secType, '')
+#               
+#         return s
+#         
+#     @staticmethod
+#     def makeRedisKeyEx(contract, old=False):
+#         # this routine is to circumvent a problem in makeRedisKey with 
+#         # the key in position 3 having different meanings under different conditions.
+#         #  
+#         # 
+#         # to differentiate the keys generated by the old and new functions, 
+#         # contract keys created using this routine have their last slot
+#         # hard coded a magic number 102
+#           
+#         if (old):
+#             return ContractHelper.makeRedisKey(contract)
+#           
+#         contract.m_strike = int(contract.m_strike)
+# # amend 10/22 
+# # add exchange to the key         
+#         s = '%s-%s-%s-%s-%s-%s-%s-%d' % (contract.m_symbol,
+#                                                            contract.m_expiry,
+#                                                            contract.m_strike,
+#                                                            contract.m_right,
+#                                                            contract.m_secType,
+#                                                            contract.m_currency,
+#                                                              
+#                                                            contract.m_exchange,
+#                                                              
+#                                                            102)
+#         return s
+
+
+
 # copy list of subscribed contracts from the console in options_data
 # save them into a text file
 # call this function to generate subscription txt that 
 # can be processed by ib_mds.py
-def create_subscription_file(src, dest):
+# def create_subscription_file(src, dest):
+#     
+#     # improper file content will cause
+#     # this function to fail
+#     f = open(src)
+#     lns = f.readlines()
+# 
+#     a= filter(lambda x: x[0] <> '\n', map(lambda x: x.split(','), lns))
+#     contracts = map(lambda x: x.split('-'), [c[1] for c in a])
+#     options = filter(lambda x: x[2] <> 'FUT', contracts)
+#     futures = filter(lambda x: x[2] == 'FUT', contracts)
+#     print contracts
+#     #HSI,FUT,HKFE,HKD,20151029,0,
+#     futm= map(lambda x: "%s,%s,%s,%s,%s,%s,%s" % (x[0], 'FUT', 'HKFE', 'HKD', x[1], '0', ''), futures)
+#     outm= map(lambda x: "%s,%s,%s,%s,%s,%s,%s" % (x[0], 'OPT', 'HKFE', 'HKD', x[1], x[2], x[3]), options)
+#     f1 = open(dest, 'w')
+#     f1.write(''.join('%s\n'% c for c in outm))
+#     f1.write(''.join('%s\n'% c for c in futm))
+#     f1.close()
+
+def create_subscription_file(dest):
     
     # improper file content will cause
     # this function to fail
-    f = open(src)
-    lns = f.readlines()
+    
+    lns = DataMap().printMapCSV().split('\n')
 
-    a= filter(lambda x: x[0] <> '\n', map(lambda x: x.split(','), lns))
+    a= filter(lambda x: x[0] <> '', map(lambda x: x.split(','), lns))
     contracts = map(lambda x: x.split('-'), [c[1] for c in a])
     options = filter(lambda x: x[2] <> 'FUT', contracts)
     futures = filter(lambda x: x[2] == 'FUT', contracts)
@@ -915,6 +981,7 @@ def create_subscription_file(src, dest):
     f1.write(''.join('%s\n'% c for c in futm))
     f1.close()
 
+
 def console(config, omd):
     undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
     done = False
@@ -922,7 +989,7 @@ def console(config, omd):
 #        try:
         print "Available commands are: l - list all subscribed contracts, i - force recal impl vol, s <H/M> <X> - H for HSI M for MHI manual subscription of HKFE options"
         print "                        p - print portfolio summary,   r - rescan portfolio and update subscription list"
-        print "                        c [src] [dest] - dump subscribed contracts to an external file"
+        print "                        c [dest] - dump subscribed contracts to an external file"
         print "                       <id> - list subscribed contract by id, q - terminate program"
         cmd = raw_input(">>")
         input = cmd.split(' ')
@@ -956,7 +1023,7 @@ def console(config, omd):
         elif input[0] == 'r':
             add_portfolio_subscription(config, omd)
         elif input[0] == 'c':
-            create_subscription_file(input[1], input[2])
+            create_subscription_file(input[1])
             
         elif isinstance(input[0], int): 
             DataMap().printContractByID(input[0])

finopt/portfolio.py

@@ -7,7 +7,7 @@ import thread, threading
 from threading import Lock
 import ConfigParser
 from ib.ext.Contract import Contract
-from ib.opt import ibConnection, message
+from ib.opt import ibConnection#, message
 from time import sleep
 import time, datetime
 import optcal
@@ -52,7 +52,7 @@ class PortfolioManager():
     grouped_options = None
     
     # item 0 is for position, item 1 is for account
-    download_states = [False, False]
+    download_states = None
     quit = False
     interested_types = ['OPT', 'FUT']
     rs_port_keys = {}
@@ -94,6 +94,7 @@ class PortfolioManager():
         self.con.registerAll(self.on_ib_message)
         
         
+    	self.download_states = [False, False]
         self.tlock = Lock()
     
     def retrieve_position(self):

finopt/portfolio_kf.py

@@ -0,0 +1,626 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import thread, threading
+from threading import Lock
+import ConfigParser
+
+from ib.ext.ExecutionFilter import ExecutionFilter
+from time import sleep
+import time, datetime
+
+
+import redis
+print sys.path
+
+
+from misc2.helpers import ContractHelper, ExecutionFilterHelper
+
+from comms.epc import EPCPub
+from comms.tws_client import SimpleTWSClient
+
+
+
+#import options_data
+
+# Tick Value      Description
+# 5001            impl vol
+# 5002            delta
+# 5003            gamma
+# 5004            theta
+# 5005            vega
+# 5006            premium
+
+# 6001            avgCost
+# 6002            pos
+# 6003            totCost
+# 6004            avgPx
+# 6005            pos delta
+# 6006            pos theta
+# 6007            multiplier
+# 6020            pos value impact +1% vol change
+# 6021            pos value impact -1% vol change
+
+
+# IB tick types code reference
+# https://www.interactivebrokers.com/en/software/api/api.htm
+                    
+
+class PortfolioManagerKF(threading.Thread):
+    
+    config = {}
+    con = None
+    r_conn = None
+    port = []
+    grouped_options = None
+    
+    # item 0 is for position, item 1 is for account
+    download_states = None
+    quit = False
+    interested_types = ['OPT', 'FUT']
+    rs_port_keys = {}
+    ib_port_msg = []
+    tlock = None
+    epc = None
+    account_tags = []
+    ib_acct_msg = {}
+    check_interval = {'exec': 5, 'port': 30}
+    has_new_execution = False
+    retrieve_position_complete = False
+    
+    def __init__(self, config):
+        super(PortfolioManagerKF, self).__init__()
+        self.config = config
+
+        host = config.get("market", "ib.gateway").strip('"').strip("'")
+        port = int(config.get("market", "ib.port"))
+        appid = int(config.get("market", "ib.appid.portfolio"))   
+        self.rs_port_keys['port_conid_set'] = config.get("redis", "redis.datastore.key.port_conid_set").strip('"').strip("'")
+        self.rs_port_keys['port_prefix'] = config.get("redis", "redis.datastore.key.port_prefix").strip('"').strip("'")        
+        self.rs_port_keys['port_summary'] = config.get("redis", "redis.datastore.key.port_summary").strip('"').strip("'")
+        self.rs_port_keys['port_items'] = config.get("redis", "redis.datastore.key.port_items").strip('"').strip("'")
+        
+        
+        self.rs_port_keys['acct_summary'] = config.get("redis", "redis.datastore.key.acct_summary").strip('"').strip("'") 
+        self.account_tags = eval(config.get("portfolio", "portfolio.account_summary_tags").strip('"').strip("'"))
+        
+        self.epc = eval(config.get("portfolio", "portfolio.epc").strip('"').strip("'"))
+        # instantiate a epc object if the config says so
+        
+        
+        if self.epc['stream_to_Kafka']:
+            self.epc['epc'] = EPCPub(config) 
+        
+        r_host = config.get("redis", "redis.server").strip('"').strip("'")
+        r_port = config.get("redis", "redis.port")
+        r_db = config.get("redis", "redis.db")             
+        
+        self.r_conn = redis.Redis(r_host, r_port, r_db)
+        
+        khost = config.get("epc", "kafka.host").strip('"').strip("'")
+        kport = config.get("epc", "kafka.port")
+        
+#        self.con = ibConnection(host, port, appid)
+        self.con = SimpleTWSClient(khost, kport)
+        self.con.registerAll([self.on_ib_message])
+        
+        
+    	self.download_states = [False, False]
+        self.tlock = Lock()
+        
+    
+    
+    def retrieve_position(self):
+        print 'this function no longer do anything.'
+        pass
+    
+    
+    def run(self):
+        
+        self.connect()
+        
+        
+        while 1:
+            now = datetime.datetime.now()
+            exec_filter = ExecutionFilterHelper.kv2object({'m_time': now.strftime('%Y%m%d  %H%M%S')}, ExecutionFilter)
+            self.con.get_command_handler().reqExecutions(exec_filter)
+            if self.has_new_execution:
+                while self.retrieve_position_complete == False:
+                    pass
+                self._retrieve_position()
+            sleep(self.check_interval['exec'])
+            
+                
+                
+    
+    def _retrieve_position(self):
+        #self.connect()
+        
+        # clear previous saved values
+        self.port = []
+        self.ib_port_msg = []
+        self.clear_redis_portfolio()
+        self.quit = False
+        self.subscribe()
+        while not self.retrieve_position_complete: 
+            if self.download_states[0] == True and self.download_states[1] == True:
+                
+                
+                print '-------------------------------'
+                print ' some lengthy operations....set the number larger than the consumer timeout to test....'
+                print self.download_states
+                #sleep(7)
+                print '-- now instruct consumer to die '
+                print  '-----------------------------'
+                
+                #self.disconnect()
+                self.retrieve_position_complete = True
+                
+    def clear_redis_portfolio(self):
+        l = map(lambda x: (self.r_conn.delete(x)), self.r_conn.keys(pattern='%s*' % (self.rs_port_keys['port_prefix'])))
+        #self.r_set(self.rs_port_keys['port_summary'], '{"Portfolio Retrieval": "Calculation in progress...Try again later!" }')
+        self.r_conn.set(self.rs_port_keys['port_summary'], '{"Portfolio Retrieval": "Calculation in progress...Try again later!" }')
+
+        logging.debug('clear_redis_portfolio: num items cleared: %d'% len(l))
+        
+    
+    def subscribe(self):
+
+            self.con.get_command_handler().reqPositions()
+            logging.debug('account info to retrieve: %s' % (''.join('%s,' % s for s in self.account_tags)))
+            #self.con.reqAccountSummary(100, 'All', ''.join('%s,' % s for s in self.account_tags))
+            self.con.get_command_handler().reqAccountSummary(100, 'All', ''.join('%s,' % s for s in self.account_tags))
+            #self.con.register(self.on_ib_message, 'UpdateAccountValue')
+
+    def on_ib_message(self, msg):
+        
+        print msg
+        
+        if msg.typeName in 'execution':
+            print 'execution-------'
+            print msg
+            self.has_new_execution = True
+        
+        if msg.typeName in "position":
+            if self.download_states[0] == False:
+                logging.debug("%s" %  (ContractHelper.printContract(msg.contract)))
+                if msg.contract.m_secType in self.interested_types:
+                    logging.debug("PortfolioManager: getting position...%s" % msg)
+                    self.ib_port_msg.append(msg)
+                    #self.construct_port(msg)
+            
+        if msg.typeName == 'positionEnd':
+            
+
+            for pm in self.ib_port_msg:
+                self.construct_port(pm)
+                
+            self.recal_port()
+            self.group_pos_by_strike()
+            logging.debug("PortfolioManager: Complete position download. Disconnecting...")
+            
+
+            self.download_states[0] = True
+            
+        if msg.typeName == "accountSummary":
+            if self.download_states[1] == False:
+                
+                self.ib_acct_msg[msg.tag] = (msg.value, msg.currency, msg.account)
+                logging.debug("PortfolioManager: getting account info...%s" % msg)
+            
+        if msg.typeName == 'accountSummaryEnd':
+            
+            self.ib_acct_msg['last_updated'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S') 
+            logging.info("-------------- ACCOUNT SUMMARY [%s]" % (self.ib_acct_msg['AccountType'][2]))
+            logging.info('\n\n' + ''.join('%30s: %22s\n' % (k, ''.join('%10s %12s' % (v[0], v[1] if v[1] else ''))) for k,v in self.ib_acct_msg.iteritems()))
+            
+            self.r_conn.set(self.rs_port_keys['acct_summary'], json.dumps(self.ib_acct_msg))
+            if self.epc['epc']:
+                try:
+
+                    self.epc['epc'].post_account_summary(self.ib_acct_msg)
+
+                except:
+                    logging.exception("Exception in function when trying to broadcast account summary message to epc")
+            
+            self.download_states[1] = True
+            
+            
+            
+            
+    def group_pos_by_strike(self):
+
+
+        # split into lines of position       
+        m = map(lambda x: x.split(','), self.port)
+        # transform each line into two elements. first one is a key created 
+        # by combining right and strike, the second is the product of 
+        # position * conversion ratio (HSI=50, MSI=10)
+        n = map(lambda x: (x[3] + x[4], float(x[5]) * float(x[6])/50), m)
+        
+        #p = dict(set(map(lambda x:(x[0], 0), n)))
+        
+        def sumByKey(f, n):
+            # filter the list with only unique keys
+            # transform the list into a dict
+            p = dict(set(map(f, n)))
+            # add the numbers together on key
+            l =[]
+            for x in n:
+                p[x[0]] += x[1]
+                
+            return [(k[1:], k[0:1],v) for k,v in p.iteritems()]
+         
+         
+           
+        #print len(n),n
+        # initialize a list of strikes and the position sum to 0
+        # pass the list to sumByKey routine
+        self.grouped_options = sumByKey(lambda x:(x[0], 0), n)
+        #print len(l), sorted(l)
+        
+    def group_pos_by_right(self):        
+    # group by put, call (summing up all contracts by right type,ignoring strikes)
+        # split into lines of position       
+        m = map(lambda x: x.split(','), self.port)
+        q = map(lambda x: (x[3], float(x[5]) * float(x[6])/50), m)
+        u = dict(set(map(lambda x:(x[0], 0.0), q)))
+#        print u, q
+        for x in q:
+            u[x[0]] += x[1] 
+        return [(a,b) for a,b in u.iteritems()]
+    
+    def get_grouped_options_str_array(self):
+        s = ''
+        for e in sorted(self.grouped_options):
+            s += "[%f,%s,%s]," % (float(e[0])/100.0, e[2] if e[1] == 'P' else 0, e[2] if e[1] == 'C' else 0)
+        return s
+    
+    
+    def get_traded_months(self):
+        
+        li = []
+        for l in self.port:
+            toks=  l.split(',')
+            li.append(toks[2])
+        #print sorted(list(set(li)))
+        return sorted(list(set(li)))
+    
+
+    def get_greeks(self, c_key):
+        #print c_key, len(c_key), self.r_conn.get('a')
+        
+        gdata = self.r_conn.get(c_key)
+        
+        #print gdata
+        gmap = json.loads(gdata) if gdata <> None else None
+        return gmap
+
+
+    def get_pos_contracts(self):
+        s_cols = [0,2,4,3]
+        cs = []
+        for l in sorted(self.port):
+            content = ''    
+            toks= l.split(',')
+            c_key = '-'.join('%s' % toks[i] for i in s_cols)
+            cs.append(c_key)
+        return cs
+
+    def get_tbl_pos_csv(self):
+
+        # WARNING: this routine will crash if a new instrument is traded during the day
+        # but it is not previously been subscribed 
+        # the gmap will be missing from redis
+        # the workaround is to restart options_data.py 
+        # to refresh all positions and add back any
+        # new instruments to its subscription list 
+
+        pall = set(self.r_conn.keys(pattern='%s*' % self.rs_port_keys['port_prefix']))
+        s = '["symbol","right","avgcost","spotpx","pos","delta","theta","pos_delta","pos_theta","unreal_pl","last_updated"],'
+        
+        def split_toks(x):
+            pmap = json.loads(self.r_conn.get(x))
+            #print pmap
+            gmap = json.loads(self.r_conn.get(x[3:]))
+            #print gmap
+            s = '["%s","%s",%f,%f,%f,%f,%f,%f,%f,%f,"%s"],' % (x[3:], x[len(x)-1:], pmap['6001'], gmap['5006'], pmap['6002'],\
+                                                                         gmap['5002'],gmap['5004'],\
+                                                                         pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated'])
+            return s                                                          
+            
+        end_s = s + ''.join (split_toks( x ) for x in pall)
+        return end_s 
+
+
+    def get_tbl_pos_csv_old(self, calGreeks=False):
+        s_cols = [0,1,2,3,4]
+        i_cols = [5,6,7]
+        s = '["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],'
+        
+        for l in sorted(self.port):
+            content = ''    
+            toks= l.split(',')
+ #           print toks
+            for i in s_cols:
+                content += '"%s",' % toks[i]
+            for i in i_cols:
+                content += '%s,' % toks[i]
+            
+                
+            s += "[%s]," % content
+        return s  
+
+    def get_portfolio_summary(self):
+
+        
+        return json.loads(self.r_conn.get(self.rs_port_keys['port_summary']))
+    
+
+    def recal_port(self):
+        # http://stackoverflow.com/questions/448034/multithreaded-resource-access-where-do-i-put-my-locks
+        
+        # a wrapper just to ensure only one thread is doing the portfolio recalculation
+        logging.debug('recal_port: acquiring lock...%s' % threading.currentThread())
+        self.tlock.acquire()
+        try:
+            s = self.recal_port_rentrant_unsafe()
+        finally:
+            logging.debug('recal_port: completed recal. releasing lock...')
+            self.tlock.release()  
+            
+        return s
+
+    def recal_port_rentrant_unsafe(self):
+        
+        
+        logging.debug('PortfolioManager: recal_port')
+
+        # retrieve the portfolio entries from redis, strip the prefix in front
+        plines = map(lambda k: (k.replace(self.rs_port_keys['port_prefix'] + '_', '')),\
+                                 self.r_conn.keys(pattern=('%s*'% self.rs_port_keys['port_prefix'])))
+                                 
+        logging.info ("PortfolioManager: recal_port-> gathering position entries from redis %s" % plines)
+        
+        pos_summary = {'delta_c': 0.0, 'delta_p': 0.0, 'delta_all': 0.0,\
+                       'theta_c': 0.0, 'theta_p': 0.0, 'theta_all': 0.0,\
+                       'delta_1percent' : 0.0, 'theta_1percent' : 0.0,\
+                       'iv_plus1p': 0.0, 'iv_minus1p': 0.0, 'unreal_pl': 0.0}
+        l_gmap = []
+        l_skipped_pos =[]       
+        t_pos_multiplier = 0.0
+        
+        for ckey in plines:
+        
+            gmap = self.get_greeks(ckey)
+            logging.debug('PortfolioManager: recal_port greeks market data %s->%s ' % (ckey, gmap))
+            logging.debug('PortfolioManager: recal_port position-map->%s' % (self.r_get(ckey)))
+            pmap = json.loads(self.r_get(ckey)) 
+            
+            
+            
+            if gmap:
+            
+            # Tick Value      Description
+            # 5001            impl vol
+            # 5002            delta
+            # 5003            gamma
+            # 5004            theta
+            # 5005            vega
+            # 5006            premium        
+            # 6001            avgCost
+            # 6002            pos
+            # 6003            totCost
+            # 6004            avgPx
+            # 6005            pos delta
+            # 6006            pos theta
+            # 6007            multiplier
+            # 6009            curr_port_value
+            # 6008            unreal_pl
+            # 6020            pos value impact +1% vol change
+            # 6021            pos value impact -1% vol change
+            
+           
+                def pos_delta():                 
+                    pd = pmap['6002'] * gmap['5002'] * pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_delta: %f' % pd)
+                    return pd
+                
+                def pos_theta():
+                    pd = pmap['6002'] * gmap['5004'] * pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_theta: %f' % pd)
+                    return pd
+ 
+                def pos_avg_px():
+                    pd = pmap['6001'] / pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_avg_px: %f' % pd)
+                    return pd                   
+                    
+                def pos_tot_cost():
+                    pd = pmap['6001'] * pmap['6002'] * pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_tot_cost: %f' % pd)
+                    return pd                   
+                
+                def pos_unreal_pl():
+                    #(spot premium * multiplier - avgcost) * pos) 
+                    v = (gmap['5006'] * pmap['6007'] - pmap['6001']) * pmap['6002'] 
+                    return v 
+    
+                
+    
+                #logging.debug('PortfolioManager: recal_port greeks %f' % pos_delta(gmap))
+                
+                pmap['6005'] = pos_delta()
+                pmap['6006'] = pos_theta()
+                pmap['6004'] = pos_avg_px()
+                pmap['6003'] = pos_tot_cost()
+                pmap['6008'] = pos_unreal_pl()
+                pmap['last_updated'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+                
+                l_gmap.append(pmap)          
+
+                #t_pos_multiplier += (pmap['6002'] * pmap['6007'])
+
+                right = ckey.split('-')[3].lower()
+                pos_summary['delta_' + right] += pmap['6005']
+                pos_summary['delta_all'] += pmap['6005']
+                pos_summary['theta_' + right] += pmap['6006']
+                pos_summary['theta_all'] += pmap['6006']
+                pos_summary['unreal_pl'] += pmap['6008']
+
+                
+                
+                #pos_summary['delta_1percent'] += (pos_summary['delta_all'] / (pmap['6002'] * pmap['6007']))
+                # delta_1% = pos_ / (pos * multiplier)
+                
+                
+                #print 'con,right,avgcost,spot px,pos,delta,theta, pos_delta,pos_theta,pos_unreal_pl,last_updated'
+#                 print ( 'PT_entries: %s,%s,%f,%f,%f,%f,%f,%f,%f,%f,%s' % (ckey, right, pmap['6001'], gmap['5006'], pmap['6002'],\
+#                                                                      gmap['5002'],gmap['5004'],\
+#                                                                      pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated']
+#                                                                      ))
+#20150911                
+                #print pmap
+                #print json.dumps(pmap)
+                self.r_set(ckey, json.dumps(pmap))
+                
+                logging.debug('PortfolioManager: update position in redis %s' % self.r_get(ckey))
+                
+            else:
+                l_skipped_pos.append(ckey)
+
+            
+#            self.r_set(ckey, json.dumps(pmap))
+#            logging.debug('PortfolioManager: update position in redis %s' % self.r_get(ckey))
+
+ 
+        #pos_summary['delta_1percent'] = (pos_summary['delta_all'] / t_pos_multiplier)
+        
+        
+
+ 
+        if len(l_skipped_pos) > 0:
+            logging.warn('***************** PortfolioManager: recal_port. SOME POSITIONS WERE NOT PROCESSED!')
+            logging.warn('----------------- DO NOT rely on the numbers in the Summary dictionary (pos_summary)')
+            logging.warn('----------------- Please check your portfolio through other means or subscribe missing')
+            logging.warn('----------------- market data in options_serve.py console. ')
+        logging.info('-------------- POSITION SUMMARY')
+        pos_summary['last_updated'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')    
+        pos_summary['entries_skipped'] = l_skipped_pos
+        pos_summary['status'] = 'OK' if len(l_skipped_pos) == 0 else 'NOT_OK'
+        #self.r_set(self.rs_port_keys['port_summary'], json.dumps(pos_summary) )
+        t_pos_summary = json.dumps(pos_summary)
+        self.r_conn.set(self.rs_port_keys['port_summary'], t_pos_summary )
+        self.r_conn.set(self.rs_port_keys['port_items'], json.dumps(l_gmap))
+        #print pos_summary
+        #print l_gmap      
+        # broadcast 
+        if self.epc['epc']:
+            try:
+                self.epc['epc'].post_portfolio_summary(pos_summary)
+                self.epc['epc'].post_portfolio_items(l_gmap)
+            except:
+                logging.exception("Exception in function: recal_port_rentrant_unsafe")
+
+        #logging.info(pos_summary)
+        
+        logging.warn('-------------- Entries for which the greeks are not computed!! %s' %\
+                        ','.join(' %s' % k for k in l_skipped_pos))
+        
+        return l_gmap
+ 
+
+    def construct_port(self, pos_msg):
+        # port structure
+        #
+        # exch,type,contract_mth,right, strike,con_ration,pos,avgcost
+        # 
+        # Tick Value      Description
+        # 5001            impl vol
+        # 5002            delta
+        # 5003            gamma
+        # 5004            theta
+        # 5005            vega
+        # 5005            premium
+        
+        # 6001            avgCost
+        # 6002            pos
+    
+        
+        toks = ContractHelper.printContract(pos_msg.contract).split('-')
+        s = ''
+        
+        slots = [0, 1, 4, 6]
+        s = s + '%s,%s,%s' % (','.join(toks[i] for i in slots), toks[5].replace('.0', ''), '50.0' if toks[0][1:] == 'HSI' else '10.0')
+        s = s.replace('[', '') + ",%0.4f,%0.4f" % (pos_msg.pos, pos_msg.avgCost)
+        
+        
+        self.port.append(s)
+                
+        ckey = ContractHelper.makeRedisKey(pos_msg.contract)
+        multiplier = 50.0 if toks[0][1:] == 'HSI' else 10.0
+
+        
+        
+        self.r_set(ckey, json.dumps({"contract": ckey, "6002": pos_msg.pos, "6001": pos_msg.avgCost, "6007": multiplier}))
+        #self.recal_port(ckey)
+ 
+    #the wrapper functions add a prefix session id to every key
+    #
+    def r_set(self, key, val):
+        return self.r_conn.set("%s_%s" % (self.rs_port_keys['port_prefix'], key), val)
+        
+    
+    def r_get(self, key):
+        return self.r_conn.get("%s_%s" % (self.rs_port_keys['port_prefix'], key))
+    
+    
+    def r_sadd(self, key, val):
+        return self.r_conn.sadd("%s_%s" % (self.rs_port_keys['port_prefix'], key), val)
+    
+    def r_sismember(self, set, val):
+        return self.r_conn.sismember("%s_%s" % (self.rs_port_keys['port_prefix'], set), val)
+
+    def r_smembers(self, key):
+        return self.r_conn.smembers("%s_%s" % (self.rs_port_keys['port_prefix'], key))
+       
+
+    def connect(self):
+        self.con.connect()
+
+        
+    def disconnect(self):
+
+        self.con.disconnect()
+        self.quit = True
+      
+
+if __name__ == '__main__':
+           
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]    
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0:      
+        raise ValueError, "Failed to open config file" 
+    
+    logconfig = eval(config.get("portfolio", "portfolio.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)        
+
+    p = PortfolioManagerKF(config)
+    p.retrieve_position()
+    p.start()
+    
+    print p.get_portfolio_summary()
+    print p.get_tbl_pos_csv()
+
+
+    
+         
+    
+

finopt/register_topics.py

@@ -0,0 +1,73 @@
+import sys
+from time import sleep, strftime
+import time, datetime
+import ConfigParser
+from optparse import OptionParser
+import logging
+import thread
+import threading
+import traceback
+import json
+from threading import Lock
+from ib.ext.Contract import Contract
+from ib.ext.EWrapper import EWrapper
+from ib.ext.EClientSocket import EClientSocket
+from ib.ext.ExecutionFilter import ExecutionFilter
+from ib.ext.Execution import Execution
+from ib.ext.OrderState import OrderState
+from ib.ext.Order import Order
+
+from kafka.client import KafkaClient
+from kafka import KafkaConsumer
+from kafka.producer import SimpleProducer
+from kafka.common import LeaderNotAvailableError
+
+from misc2.helpers import ContractHelper, OrderHelper, ExecutionFilterHelper
+from comms.ib_heartbeat import IbHeartBeat
+from comms.tws_protocol_helper import TWS_Protocol 
+
+class KTopicsManager:
+    
+    def __init__(self, host, port):
+        client = KafkaClient('%s:%s' % (host, port))
+        self.producer = SimpleProducer(client, async=False)
+        self.cli_request_handler = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.topicMethods) + list(TWS_Protocol.gatewayMethods) ], \
+                                   metadata_broker_list=['%s:%s' % (host, port)],\
+                                   group_id = 'epc.tws_gateway',\
+                                   auto_commit_enable=True,\
+                                   auto_commit_interval_ms=30 * 1000,\
+                                   auto_offset_reset='largest')        
+
+    def register(self, topics):
+        for t in topics:
+            try:
+                print ('registering %s' % t)
+                self.producer.send_messages(t, 'register topic - %s' % t)
+            except:
+                continue
+
+
+             
+             
+def register_topics(host, port):
+    
+        
+    reg = KTopicsManager(host, port)
+    reg.register(TWS_Protocol.topicEvents)
+    reg.register(TWS_Protocol.topicMethods)    
+    reg.register(TWS_Protocol.gatewayEvents)
+    reg.register(TWS_Protocol.gatewayMethods)
+    reg.register(TWS_Protocol.oceEvents)
+    #reg.register(TWS_Protocol.oceMethods)
+    
+    
+    
+
+    
+if __name__ == '__main__':    
+          
+    
+     
+       
+    register_topics('vsu-01', 9092)
+             

finopt/test2.py

@@ -296,6 +296,31 @@ def analyze():
     ee = filter(lambda x: x <> None, dd)
     print '\n'.join('%s %2f' % (x[0], x[1]) for x in ee)
 
+
+from ws4py.client.threadedclient import WebSocketClient
+
+class DummyClient(WebSocketClient):
+    def opened(self):
+        def data_provider():
+            for i in range(1, 200, 25):
+                yield "#" * i
+
+        self.send(data_provider())
+
+        for i in range(0, 200, 25):
+            print i
+            self.send("*" * i)
+
+    def closed(self, code, reason=None):
+        print "Closed down", code, reason
+
+    def received_message(self, m):
+        print m
+        if len(m) == 175:
+            self.close(reason='Bye bye')
+
+
+
     
     
 if __name__ == '__main__':
@@ -332,7 +357,15 @@ if __name__ == '__main__':
 #    analyze()
 
 
-    stdan2('/home/larry/l1304/workspace/finopt/data/mds_files/std/', 'std-20151007')
+#    stdan2('/home/larry/l1304/workspace/finopt/data/mds_files/std/', 'std-20151007')
+
+
+    try:
+        ws = DummyClient('ws://localhost:8082/ws', protocols=['http-only', 'chat'])
+        ws.connect()
+        ws.run_forever()
+    except KeyboardInterrupt:
+        ws.close()
     
     
 

finopt/testkcon.py

@@ -0,0 +1,487 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+import json
+import logging
+import ConfigParser
+from time import sleep
+import time, datetime
+import sleekxmpp
+from threading import Lock
+from kafka.client import KafkaClient
+from kafka import KafkaConsumer
+from kafka.producer import SimpleProducer
+from kafka.common import LeaderNotAvailableError
+import threading
+from options_data import ContractHelper
+
+
+        
+class KConsumer(threading.Thread):
+    
+    producer = None
+    consumer = None
+    kwrapper = None
+
+    def __init__(self, config):
+
+        super(KConsumer, self).__init__()
+        host = config.get("epc", "kafka.host").strip('"').strip("'")
+        port = config.get("epc", "kafka.port")
+
+        client = KafkaClient('%s:%s' % (host, port))
+        self.producer = SimpleProducer(client, async=False)
+        
+        topicsEvents = ['accountDownloadEnd', 'execDetailsEnd', 'updateAccountTime', 'deltaNeutralValidation', 'orderStatus',\
+                  'updateAccountValue', 'historicalData', 'openOrderEnd', 'updatePortfolio', 'managedAccounts', 'contractDetailsEnd',\
+                  'positionEnd', 'bondContractDetails', 'accountSummary', 'updateNewsBulletin', 'scannerParameters', \
+                  'tickString', 'accountSummaryEnd', 'scannerDataEnd', 'commissionReport', 'error', 'tickGeneric', 'tickPrice', \
+                  'nextValidId', 'openOrder', 'realtimeBar', 'contractDetails', 'execDetails', 'tickOptionComputation', \
+                  'updateMktDepth', 'scannerData', 'currentTime', 'error_0', 'error_1', 'tickSnapshotEnd', 'tickSize', \
+                  'receiveFA', 'connectionClosed', 'position', 'updateMktDepthL2', 'fundamentalData', 'tickEFP']        
+        
+ 
+        self.consumer = KafkaConsumer( *[(v,0) for v in topicsEvents], \
+                                       metadata_broker_list=['%s:%s' % (host, port)],\
+                                       group_id = 'epc.group',\
+                                       auto_commit_enable=True,\
+                                       auto_commit_interval_ms=30 * 1000,\
+                                       auto_offset_reset='largest') # discard old ones
+        
+        self.kwrapper= KWrapper(self.producer)
+
+    def showmessage(self, msg_name, mapping):
+        try:
+            mapping = json.loads(mapping)
+            del(mapping['self'])
+        except (KeyError, ):
+            pass
+        items = list(mapping.items())
+        items.sort()
+        print(('### %s' % (msg_name, )))
+        for k, v in items:
+            print(('    %s:%s' % (k, v)))
+        #print mapping.items()
+        #print mapping
+        m = Message(**mapping)
+        print m
+        
+
+    def accountDownloadEnd(self, items):
+        self.showmessage("accountDownloadEnd", items)
+
+    def execDetailsEnd(self, items):
+        self.showmessage("execDetailsEnd", items)
+
+    def updateAccountTime(self, items):
+        self.showmessage("updateAccountTime", items)
+
+    def deltaNeutralValidation(self, items):
+        self.showmessage("deltaNeutralValidation", items)
+
+    def orderStatus(self, items):
+        self.showmessage("orderStatus", items)
+
+    def updateAccountValue(self, items):
+        self.showmessage("updateAccountValue", items)
+
+    def historicalData(self, items):
+        self.showmessage("historicalData", items)
+
+    def openOrderEnd(self, items):
+        self.showmessage("openOrderEnd", items)
+
+    def updatePortfolio(self, items):
+        self.showmessage("updatePortfolio", items)
+
+    def managedAccounts(self, items):
+        self.showmessage("managedAccounts", items)
+
+    def contractDetailsEnd(self, items):
+        self.showmessage("contractDetailsEnd", items)
+
+    def positionEnd(self, items):
+        self.showmessage("positionEnd", items)
+
+    def bondContractDetails(self, items):
+        self.showmessage("bondContractDetails", items)
+
+    def accountSummary(self, items):
+        self.showmessage("accountSummary", items)
+
+    def updateNewsBulletin(self, items):
+        self.showmessage("updateNewsBulletin", items)
+
+    def scannerParameters(self, items):
+        self.showmessage("scannerParameters", items)
+
+    def tickString(self, items):
+        self.showmessage("tickString", items)
+
+    def accountSummaryEnd(self, items):
+        self.showmessage("accountSummaryEnd", items)
+
+    def scannerDataEnd(self, items):
+        self.showmessage("scannerDataEnd", items)
+
+    def commissionReport(self, items):
+        self.showmessage("commissionReport", items)
+
+    def error(self, items):
+        self.showmessage("error", items)
+
+    def tickGeneric(self, items):
+        self.showmessage("tickGeneric", items)
+
+    def tickPrice(self, items):
+        self.showmessage("tickPrice", items)
+
+    def nextValidId(self, items):
+        self.showmessage("nextValidId", items)
+
+    def openOrder(self, items):
+        self.showmessage("openOrder", items)
+
+    def realtimeBar(self, items):
+        self.showmessage("realtimeBar", items)
+
+    def contractDetails(self, items):
+        self.showmessage("contractDetails", items)
+
+    def execDetails(self, items):
+        self.showmessage("execDetails", items)
+
+    def tickOptionComputation(self, items):
+        self.showmessage("tickOptionComputation", items)
+
+    def updateMktDepth(self, items):
+        self.showmessage("updateMktDepth", items)
+
+    def scannerData(self, items):
+        self.showmessage("scannerData", items)
+
+    def currentTime(self, items):
+        self.showmessage("currentTime", items)
+
+    def error_0(self, items):
+        self.showmessage("error_0", items)
+
+    def error_1(self, items):
+        self.showmessage("error_1", items)
+
+    def tickSnapshotEnd(self, items):
+        self.showmessage("tickSnapshotEnd", items)
+
+    def tickSize(self, items):
+        self.showmessage("tickSize", items)
+
+    def receiveFA(self, items):
+        self.showmessage("receiveFA", items)
+
+    def connectionClosed(self, items):
+        self.showmessage("connectionClosed", items)
+
+    def position(self, items):
+        self.showmessage("position", items)
+
+    def updateMktDepthL2(self, items):
+        self.showmessage("updateMktDepthL2", items)
+
+    def fundamentalData(self, items):
+        self.showmessage("fundamentalData", items)
+
+    def tickEFP(self, items):
+        self.showmessage("tickEFP", items)
+
+
+
+
+
+#     def on_tickPrice(self, tickerId, field, price, canAutoExecute):
+#         self.showmessage('tickPrice', vars())
+# 
+#     def on_tickSize(self, tickerId, field, size):
+#         self.showmessage('tickSize', vars())
+# 
+#     def on_tickOptionComputation(self, tickerId, field, impliedVol, delta, optPrice, pvDividend, gamma, vega, theta, undPrice):
+#         self.showmessage('tickOptionComputation', vars())
+# 
+#     def on_tickGeneric(self, tickerId, tickType, value):
+#         self.showmessage('tickGeneric', vars())
+# 
+#     def on_tickString(self, tickerId, tickType, value):
+#         self.showmessage('tickString', vars())
+# 
+#     def on_tickEFP(self, tickerId, tickType, basisPoints, formattedBasisPoints, impliedFuture, holdDays, futureExpiry, dividendImpact, dividendsToExpiry):
+#         self.showmessage('tickEFP', vars())
+# 
+#     def on_orderStatus(self, orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice, clientId, whyHeId):
+#         self.showmessage('orderStatus', vars())
+# 
+#     def on_openOrder(self, orderId, contract, order, state):
+#         self.showmessage('openOrder', vars())
+# 
+#     def on_openOrderEnd(self):
+#         self.showmessage('openOrderEnd', vars())
+# 
+#     def on_updateAccountValue(self, key, value, currency, accountName):
+#         self.showmessage('updateAccountValue', vars())
+# 
+#     def on_updatePortfolio(self, contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName):
+#         self.showmessage('updatePortfolio', vars())
+# 
+#     def on_updateAccountTime(self, timeStamp):
+#         self.showmessage('updateAccountTime', vars())
+# 
+#     def on_accountDownloadEnd(self, accountName):
+#         self.showmessage('accountDownloadEnd', vars())
+# 
+#     def on_nextValidId(self, orderId):
+#         self.showmessage('nextValidId', vars())
+# 
+#     def on_contractDetails(self, reqId, contractDetails):
+#         self.showmessage('contractDetails', vars())
+# 
+#     def on_contractDetailsEnd(self, reqId):
+#         self.showmessage('contractDetailsEnd', vars())
+# 
+#     def on_bondContractDetails(self, reqId, contractDetails):
+#         self.showmessage('bondContractDetails', vars())
+# 
+#     def on_execDetails(self, reqId, contract, execution):
+#         self.showmessage('execDetails', vars())
+# 
+#     def on_execDetailsEnd(self, reqId):
+#         self.showmessage('execDetailsEnd', vars())
+# 
+#     def on_connectionClosed(self):
+#         self.showmessage('connectionClosed', {})
+# 
+#     def on_error(self, id=None, errorCode=None, errorMsg=None):
+#         self.showmessage('error', vars())
+# 
+#     def on_error_0(self, strvalue=None):
+#         self.showmessage('error_0', vars())
+# 
+#     def on_error_1(self, id=None, errorCode=None, errorMsg=None):
+#         self.showmessage('error_1', vars())
+# 
+#     def on_updateMktDepth(self, tickerId, position, operation, side, price, size):
+#         self.showmessage('updateMktDepth', vars())
+# 
+#     def on_updateMktDepthL2(self, tickerId, position, marketMaker, operation, side, price, size):
+#         self.showmessage('updateMktDepthL2', vars())
+# 
+#     def on_updateNewsBulletin(self, msgId, msgType, message, origExchange):
+#         self.showmessage('updateNewsBulletin', vars())
+# 
+#     def on_managedAccounts(self, accountsList):
+#         self.showmessage('managedAccounts', vars())
+# 
+#     def on_receiveFA(self, faDataType, xml):
+#         self.showmessage('receiveFA', vars())
+# 
+#     def on_historicalData(self, reqId, date, open, high, low, close, volume, count, WAP, hasGaps):
+#         self.showmessage('historicalData', vars())
+# 
+#     def on_scannerParameters(self, xml):
+#         self.showmessage('scannerParameters', vars())
+# 
+#     def on_scannerData(self, reqId, rank, contractDetails, distance, benchmark, projection, legsStr):
+#         self.showmessage('scannerData', vars())
+# 
+# 
+#     def on_commissionReport(self, commissionReport):
+#         self.showmessage('commissionReport', vars())
+# 
+# 
+#     def on_currentTime(self, time):
+#         self.showmessage('currentTime', vars())
+# 
+#     def on_deltaNeutralValidation(self, reqId, underComp):
+#         self.showmessage('deltaNeutralValidation', vars())
+# 
+# 
+#     def on_fundamentalData(self, reqId, data):
+#         self.showmessage('fundamentalData', vars())
+# 
+#     def on_marketDataType(self, reqId, marketDataType):
+#         self.showmessage('marketDataType', vars())
+# 
+# 
+#     def on_realtimeBar(self, reqId, time, open, high, low, close, volume, wap, count):
+#         self.showmessage('realtimeBar', vars())
+# 
+#     def on_scannerDataEnd(self, reqId):
+#         self.showmessage('scannerDataEnd', vars())
+# 
+# 
+# 
+#     def on_tickSnapshotEnd(self, reqId):
+#         self.showmessage('tickSnapshotEnd', vars())
+# 
+# 
+#     def on_position(self, account, contract, pos, avgCost):
+#         self.showmessage('position', vars())
+# 
+#     def on_positionEnd(self):
+#         self.showmessage('positionEnd', vars())
+# 
+#     def on_accountSummary(self, reqId, account, tag, value, currency):
+#         self.showmessage('accountSummary', vars())
+# 
+#     def on_accountSummaryEnd(self, reqId):
+#         self.showmessage('accountSummaryEnd', vars())
+
+
+
+                
+
+
+                            
+    def run(self):
+        
+        for message in self.consumer:
+            
+            logging.info("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+                                         message.offset, message.key,
+                                         message.value))
+
+            print ("received %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
+                                         message.offset, message.key,
+                                         message.value))
+            
+            #self.on_tickPrice(message.value)
+            getattr(self, message.topic, None)(message.value)
+            
+
+
+
+class Message(object):
+    """ Base class for Message types.
+
+    """
+
+    
+
+    def __init__(self, **kwds):
+        """ Constructor.
+
+        @param **kwds keywords and values for instance
+        """
+        
+        
+        for name in kwds.keys():
+            setattr(self, name, kwds[name])
+        
+        
+        #assert not kwds
+
+    def __len__(self):
+        """ x.__len__() <==> len(x)
+
+        """
+        return len(self.keys())
+
+    def __str__(self):
+        """ x.__str__() <==> str(x)
+
+        """
+        name = self.typeName
+        items = str.join(', ', ['%s=%s' % item for item in self.items()])
+        return '<%s%s>' % (name, (' ' + items) if items else '')
+
+    def items(self):
+        """ List of message (slot, slot value) pairs, as 2-tuples.
+
+        @return list of 2-tuples, each slot (name, value)
+        """
+        return zip(self.keys(), self.values())
+
+    def values(self):
+        """ List of instance slot values.
+
+        @return list of each slot value
+        """
+        return [getattr(self, key, None) for key in self.keys()]
+
+    def keys(self):
+        """ List of instance slots.
+
+        @return list of each slot.
+        """
+        return self.__dict__
+
+
+class KWrapper():
+    
+    producer = None
+    def __init__(self, producer):
+        self.producer = producer
+        
+    def reqAccountUpdates(self):
+        
+        self.post_msg('reqAccountUpdates', '1')
+
+    def reqExecutions(self):
+        
+        self.post_msg('reqExecutions', '')
+
+
+    def reqMktData(self, contract):
+        self.post_msg('reqMktData', ContractHelper.contract2mapstring(contract))
+        
+        
+    def post_msg(self, topic, msg):
+        logging.info('post_msg %s' % msg)
+        self.producer.send_messages(topic, msg)
+
+
+def dummy():
+    topicsEvents = ['accountDownloadEnd', 'execDetailsEnd', 'updateAccountTime', 'deltaNeutralValidation', 'orderStatus',\
+          'updateAccountValue', 'historicalData', 'openOrderEnd', 'updatePortfolio', 'managedAccounts', 'contractDetailsEnd',\
+          'positionEnd', 'bondContractDetails', 'accountSummary', 'updateNewsBulletin', 'scannerParameters', \
+          'tickString', 'accountSummaryEnd', 'scannerDataEnd', 'commissionReport', 'error', 'tickGeneric', 'tickPrice', \
+          'nextValidId', 'openOrder', 'realtimeBar', 'contractDetails', 'execDetails', 'tickOptionComputation', \
+          'updateMktDepth', 'scannerData', 'currentTime', 'error_0', 'error_1', 'tickSnapshotEnd', 'tickSize', \
+          'receiveFA', 'connectionClosed', 'position', 'updateMktDepthL2', 'fundamentalData', 'tickEFP']
+
+    s = ''.join('\tdef %s(self, items):\n\t\tself.showmessage("%s", items)\n\n' % (s,s) for s in topicsEvents)
+    return s
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0: 
+        raise ValueError, "Failed to open config file" 
+      
+    logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s %(levelname)s %(message)s')
+
+    e = KConsumer(config)
+    e.start()
+    e.kwrapper.reqAccountUpdates()
+    e.kwrapper.reqExecutions()
+    #USD,CASH,IDEALPRO,JPY,,0,
+    contractTuple = ('USD', 'CASH', 'IDEALPRO', 'JPY', '', 0, '')
+    c = ContractHelper.makeContract(contractTuple)   
+    
+    e.kwrapper.reqMktData(c)
+    contractTuple = ('USO', 'STK', 'SMART', 'USD', '', 0, '')
+    c = ContractHelper.makeContract(contractTuple)   
+    e.kwrapper.reqMktData(c)
+    
+#    print dummy()
+#     kwargs = {"arg3": 3, "arg2": "two","arg1":5}
+#     m = Message(**kwargs)
+#     print m.items()
+#     print m.arg3
+    
+    
+
+    

html/opt-chains-ex-tmpl.html

@@ -178,7 +178,7 @@
             
             $.ajax({
                 type: 'Post',
-                url: '/ws_market_data?r_ckey=HSI-20150929-FUT-&fid=4',
+                url: '/ws_market_data?r_ckey=HSI-{{{thisContractMonth}}}-FUT-&fid=4',
                 success: function (data) {
 			document.getElementById('undly_last_px').value = data;
                 }

html/opt-chains-ex-tmpl.html~

@@ -29,8 +29,8 @@
         curveType:'function',
         intervals: { 'color':'series-color' },
         interval: {
-            'i0': { 'color': '#4374E0', 'style':'bars', 'barWidth':0, 'lineWidth':1, 'pointSize':6, 'fillOpacity':0.3 },
-            'i1': { 'color': '#E49307', 'style':'bars', 'barWidth':0, 'lineWidth':1, 'pointSize':6, 'fillOpacity':0.3 },
+            'i3': { 'color': '#4374E0', 'style':'bars', 'barWidth':0, 'lineWidth':1, 'pointSize':6, 'fillOpacity':0.3 },
+            'i4': { 'color': '#E49307', 'style':'bars', 'barWidth':0, 'lineWidth':1, 'pointSize':6, 'fillOpacity':0.3 },
             'i2': { 'style':'area', 'curveType':'function', 'fillOpacity':0.3 }},
 
 	  pointShape: 'diamond',

html/opt-chains-tmpl.html

@@ -171,7 +171,7 @@
             
             $.ajax({
                 type: 'Post',
-                url: '/ws_market_data?r_ckey=HSI-20151029-FUT-&fid=4',
+                url: '/ws_market_data?r_ckey=HSI-{{{thisContractMonth}}}-FUT-&fid=4',
                 success: function (data) {
 			document.getElementById('undly_last_px').value = data;
                 }

html/opt-chains-tmpl.html~

@@ -4,11 +4,9 @@
   <title> Trading Options Monitor </title>
     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
     <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/mathjs/2.1.1/math.min.js"></script>
-
     <script type="text/javascript">
-//      google.load("visualization", "1", {packages:["corechart"]});
-//      google.load('visualization', '1', {'packages':['table', 'corehart']});
-	google.load("visualization", "1.1", {packages:["corechart", 'table']});
+      google.load("visualization", "1", {packages:["corechart"]});
+      google.load('visualization', '1', {'packages':['table']});
       google.setOnLoadCallback(drawChart);
       function drawChart() {
 	// data contains implied vols for a series of this and next month options
@@ -173,7 +171,7 @@
             
             $.ajax({
                 type: 'Post',
-                url: '/ws_market_data?r_ckey=HSI-20151029-FUT-&fid=4',
+                url: '/ws_market_data?r_ckey=HSI-{{{thisContractMonth}}}-FUT-&fid=4',
                 success: function (data) {
 			document.getElementById('undly_last_px').value = data;
                 }
@@ -214,71 +212,42 @@
             });
 
 
+
         })
     })
     </script>
-    <script>
-$(document).ready(function() {
-
-      google.setOnLoadCallback(drawChart);
-//	$(function() {
-//		$( "#tabs" ).tabs();
-//	});
-
-});
-
-
-    </script>
 
   </head>
   <body>
-
-<div id="tabs">
-  <ul>
-    <li><a href="#tabs-1">Volatility Curves</a></li>
-    <li><a href="#tabs-2">Implied Volatility Table</a></li>
-    <li><a href="#tabs-3">Risk Distribution</a></li>
-  </ul>
-  <div id="tabs-1">
         <div id="d_undly_last_px">
             <input type="button" id="b_refresh" value="Refresh Price" />
 	    <output id=undly_last_px></output>
 	    <output id=ssidx_px></output>
         </div>
-        <div id="chartLinear" style="height: 350px; width: 850px"></div>
-        <div id="chartPremium" style="height: 350px; width: 850px"></div>
-	<li>
-	<label for=haxis_from>H-Axis Range Starts At:</label>
-	<input type=range id=haxis_range min=10000 value=20000 max=25000 step=500>
-	<output id=haxis_from_value></output>
-	</li>
-
-  </div>
-  <div id="tabs-2">
-	<h2> Implied volatilities for this and next month HSI options</h2>
-	    <div id='chartTbl_div' style='width: 900px; height: 500px;'></div>
-	    <select id="format-select">
-	      <option value="">none</option>
-	      <option value="Diamond" selected>Diamond</option>
-	      <option value="triangle">triangle</option>
-	      <option value="square">square</option>
-	      <option value="diamond">diamond</option>
-	      <option value="star">star</option>
-	      <option value="polygon">polygon</option>
-	    </select>
-	    <div id="number_format_chart">
-	<li>
-	<label for=vol_divider>Volatility Divider</label>
-	<input type=range id=vol_divider min=0 value=50 max=100 step=5>
-	<output id=vol_divider_value></output>
-	</li>
-    
-  </div>
-  <div id="tabs-3">
-  </div>
-</div>
-
-
+    <div id="chartLinear" style="height: 350px; width: 850px"></div>
+    <div id="chartPremium" style="height: 350px; width: 850px"></div>
+<li>
+<label for=haxis_from>H-Axis Range Starts At:</label>
+<input type=range id=haxis_range min=10000 value=20000 max=25000 step=500>
+<output id=haxis_from_value></output>
+</li>
+<h2> Implied volatilities for this and next month HSI options</h2>
+    <div id='chartTbl_div' style='width: 900px; height: 500px;'></div>
+    <select id="format-select">
+      <option value="">none</option>
+      <option value="Diamond" selected>Diamond</option>
+      <option value="triangle">triangle</option>
+      <option value="square">square</option>
+      <option value="diamond">diamond</option>
+      <option value="star">star</option>
+      <option value="polygon">polygon</option>
+    </select>
+    <div id="number_format_chart">
+<li>
+<label for=vol_divider>Volatility Divider</label>
+<input type=range id=vol_divider min=0 value=50 max=100 step=5>
+<output id=vol_divider_value></output>
+</li>
 
 
 

html/wstest.html

@@ -0,0 +1,54 @@
+<!DOCTYPE HTML>
+<html>
+   <head>
+	
+      <script type="text/javascript">
+         function WebSocketTest()
+         {
+            if ("WebSocket" in window)
+            {
+               console.log("WebSocket is supported by your Browser!");
+               
+               // Let us open a web socket
+               var ws = new WebSocket("ws://localhost:8082/ws");
+				
+               ws.onopen = function()
+               {
+                  // Web Socket is connected, send data using send()
+                  ws.send("Message to send");
+                  console.log("Message is sent...");
+               };
+				
+               ws.onmessage = function (evt) 
+               { 
+                  var received_msg = evt.data;
+                  console.log("Message is received..." + received_msg);
+               };
+				
+               ws.onclose = function()
+               { 
+                  // websocket is closed.
+                  console.log("Connection is closed..."); 
+               };
+
+	       setInterval(function(){ ws.send("Hello" + Date.now().toString()); }, 3000);
+
+            }
+            
+            else
+            {
+               // The browser doesn't support WebSocket
+               console.log("WebSocket NOT supported by your Browser!");
+            }
+         }
+      </script>
+		
+   </head>
+   <body>
+   
+      <div id="sse">
+         <a href="javascript:WebSocketTest()">Run WebSocket</a>
+      </div>
+      
+   </body>
+</html>

html/wstest.html~

@@ -0,0 +1,54 @@
+<!DOCTYPE HTML>
+<html>
+   <head>
+	
+      <script type="text/javascript">
+         function WebSocketTest()
+         {
+            if ("WebSocket" in window)
+            {
+               console.log("WebSocket is supported by your Browser!");
+               
+               // Let us open a web socket
+               var ws = new WebSocket("ws://localhost:8082/ws");
+				
+               ws.onopen = function()
+               {
+                  // Web Socket is connected, send data using send()
+                  ws.send("Message to send");
+                  console.log("Message is sent...");
+               };
+				
+               ws.onmessage = function (evt) 
+               { 
+                  var received_msg = evt.data;
+                  console.log("Message is received..." + received_msg);
+               };
+				
+               ws.onclose = function()
+               { 
+                  // websocket is closed.
+                  console.log("Connection is closed..."); 
+               };
+
+	       setInterval(function(){ ws.send("Hello"); }, 3000);
+
+            }
+            
+            else
+            {
+               // The browser doesn't support WebSocket
+               console.log("WebSocket NOT supported by your Browser!");
+            }
+         }
+      </script>
+		
+   </head>
+   <body>
+   
+      <div id="sse">
+         <a href="javascript:WebSocketTest()">Run WebSocket</a>
+      </div>
+      
+   </body>
+</html>

misc2/helpers.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+
+
+import json
+import logging
+import threading
+from ib.ext.Contract import Contract
+from ib.ext.Order import Order
+from ib.ext.ExecutionFilter import ExecutionFilter
+
+
+class BaseHelper():
+    @staticmethod
+    def object2kvstring(o):
+        return json.dumps(o.__dict__)
+        
+    @staticmethod
+    def kvstring2object(kvstring, object):
+        return BaseHelper.kv2object(json.loads(kvstring), object)
+
+
+    @staticmethod
+    def kv2object(kv, Object):   
+        o = Object()
+        map(lambda x: o.__setattr__(x, kv[x].encode('ascii') if type(kv[x]) == unicode else kv[x]), kv.keys())
+        return o
+        
+        
+class OrderHelper(BaseHelper):
+    pass
+     
+#     @staticmethod
+#     def object2kvstring(contract):
+#         return json.dumps(contract.__dict__)
+#  
+#     @staticmethod
+#     def kvstring2object(sm_order):
+#         return OrderHelper.kv2object(json.loads(sm_order))
+#  
+#  
+#     @staticmethod
+#     def kv2object(m_order):   
+#         newOrder = Order()
+#         map(lambda x: newOrder.__setattr__(x, m_order[x].encode('ascii') if type(m_order[x]) == unicode else m_order[x]), m_order.keys())
+#         return newOrder
+
+    
+class ExecutionFilterHelper(BaseHelper):
+    pass
+
+    
+
+
+class ContractHelper(BaseHelper):
+    
+    def __init__(self, contractTuple):
+        self.makeContract(contractTuple)
+    
+    
+    @staticmethod
+    def makeContract(contractTuple):
+        newContract = Contract()
+        newContract.m_symbol = contractTuple[0]
+        newContract.m_secType = contractTuple[1]
+        newContract.m_exchange = contractTuple[2]
+        newContract.m_currency = contractTuple[3]
+        newContract.m_expiry = contractTuple[4]
+        newContract.m_strike = contractTuple[5]
+        newContract.m_right = contractTuple[6]
+        logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s:' % contractTuple)
+        return newContract
+    
+    @staticmethod
+    def convert2Tuple(newContract):
+        newContractTuple = (newContract.m_symbol,\
+                            newContract.m_secType,\
+                            newContract.m_exchange,\
+                            newContract.m_currency,\
+                            newContract.m_expiry,\
+                            newContract.m_strike,\
+                            newContract.m_right, newContract.m_conId)
+        logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s %s:' % newContractTuple)
+        return newContractTuple
+    
+
+    @staticmethod
+    def contract2kvstring(contract):
+        return json.dumps(contract.__dict__)
+
+
+
+    @staticmethod
+    def kvstring2contract(sm_contract):
+        
+        return ContractHelper.kv2contract(json.loads(sm_contract))
+
+
+    @staticmethod
+    def kv2contract(m_contract):
+        
+        newContract = Contract()
+        map(lambda x: newContract.__setattr__(x, m_contract[x].encode('ascii') if type(m_contract[x]) == unicode else m_contract[x]), m_contract.keys())
+        return newContract
+    
+
+
+    
+    @staticmethod
+    def printContract(contract):
+        s = '[%s-%s-%s-%s-%s-%s-%s-%s]' % (contract.m_symbol,
+                                                       contract.m_secType,
+                                                       contract.m_exchange,
+                                                       contract.m_currency,
+                                                       contract.m_expiry,
+                                                       contract.m_strike,
+                                                       contract.m_right,
+                                                       contract.m_conId)
+        #logging.info(s)
+        return s
+    
+    @staticmethod
+    def makeRedisKey(contract):
+        #print "makerediskey %s" % ContractHelper.printContract(contract)
+#20150904        
+        #contract.m_strike = int(contract.m_strike)
+        contract.m_strike = contract.m_strike
+        
+        if contract.m_secType == 'OPT':
+            s = '%s-%s-%s-%s' % (contract.m_symbol,
+                                                           contract.m_expiry,
+                                                           contract.m_strike,
+                                                           contract.m_right)
+        else:
+            s = '%s-%s-%s-%s' % (contract.m_symbol,
+                                                           contract.m_expiry,
+                                                           contract.m_secType, '')
+            
+        return s
+      
+    @staticmethod
+    def makeRedisKeyEx(contract, old=False):
+        # this routine is to circumvent a problem in makeRedisKey with 
+        # the key in position 3 having different meanings under different conditions.
+        #  
+        # 
+        # to differentiate the keys generated by the old and new functions, 
+        # contract keys created using this routine have their last slot
+        # hard coded a magic number 102
+        
+        if (old):
+            return ContractHelper.makeRedisKey(contract)
+        
+        #contract.m_strike = int(contract.m_strike)
+        contract.m_strike = contract.m_strike
+# amend 10/22 
+# add exchange to the key         
+#         s = '%s-%s-%s-%s-%s-%s-%s-%d' % (contract.m_symbol,
+#                                                            contract.m_expiry,
+#                                                            contract.m_strike,
+#                                                            contract.m_right,
+#                                                            contract.m_secType,
+#                                                            contract.m_currency,
+#                                                             
+#                                                            contract.m_exchange,
+#                                                             
+#                                                            102)
+        
+# amend 12/1
+#change strike format to 2 dp        
+        s = '%s-%s-%.2f-%s-%s-%s-%s-%d' % (contract.m_symbol,
+                                                           contract.m_expiry,
+                                                           float(contract.m_strike),
+                                                           contract.m_right,
+                                                           contract.m_secType,
+                                                           contract.m_currency,
+                                                            
+                                                           contract.m_exchange,
+                                                            
+                                                           102)
+        
+        return s
+    
+    
+
+    
+# def str2dict(s):
+#     return ast.literal_eval(s)
+
+def dict2str(dict):
+    # enclose strings in double quotes
+    return '{'  + ', '.join('"%s" : %s' % (k, '"%s"' % v if type(v) == str else v) for k, v in dict.iteritems()) + '}'   
+    
+
+

sh (prod)/alert.sh~

@@ -0,0 +1,5 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME/src:$PYTHONPATH
+python $FINOPT_HOMEs/src/comms/alert_bot.py $FINOPT_HOME/config/app.cfg
+

sh (prod)/md_std.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+OPTIONS="--driver-memory 2g"
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit   --driver-memory 2g --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/md_std.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+

sh (prod)/md_std2.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+OPTIONS="--driver-memory 2g"
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit   $OPTIONS --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/md_std2.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+

sh (prod)/momentum.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/momentum2.py vsu-01:2181 hsi 1 persist 
+

sh (prod)/momentum.sh~

@@ -0,0 +1,11 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$ROOT:$PYTHONPATH
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/momentum2.py vsu-01:2181 hsi 1 persist 
+

sh (prod)/momentum2.sh

@@ -0,0 +1,4 @@
+export PYTHONPATH=/home/larry-13.04/workspace/finopt:$PYTHONPATH
+KAFKA_ASSEMBLY_JAR=/home/larry-13.04/workspace/finopt/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+spark-submit --executor-cores 3 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/pairs_corr_redis.py vsu-01:2181 jpy  
+

sh (prod)/pairs_corr_redis.sh

@@ -0,0 +1,4 @@
+export PYTHONPATH=/home/larry-13.04/workspace/finopt:$PYTHONPATH
+KAFKA_ASSEMBLY_JAR=/home/larry-13.04/workspace/finopt/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+spark-submit  --jars  /home/larry-13.04/workspace/finopt/spark-streaming-kafka-assembly_2.10-1.4.1.jar /home/larry-13.04/workspace/finopt/cep/pairs_corr_redis.py vsu-01:2181 hkd-aud 
+

sh (prod)/port_stream.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+OPTIONS="--driver-memory 2g"
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit   $OPTIONS --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/port_stream.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+
+spark-submit   $OPTIONS --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/port_stream.py $SRC/config/cep.cfg vsu-01:2181 hsi 1 cal_trend 

sh (prod)/portfolio.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=/home/larry/l1304/workspace/finopt/src
+export PYTHONPATH=$ROOT
+python $ROOT/finopt/portfolio.py $ROOT/config/app.cfg

sh (prod)/readme.1st

@@ -0,0 +1,10 @@
+1. set up the environment variable
+	- open ~/.bashrc
+	- export FINOPT_HOME=/home/<user>/worker/finopt
+	- close the terminal and open again, check the
+	  environment variable is set correctly
+
+
+2. open each sh file, amend the path settings accordingly
+
+	

sh (prod)/run_mds.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+# real time mode
+python $FINOPT_HOME/cep/ib_mds.py $FINOPT_HOME/config/mds.cfg
+# replay mode
+#python $FINOPT_HOME/cep/ib_mds.py -r $FINOPT_HOME/../data/mds_files/20151006 $FINOPT_HOME/config/mds.cfg
+

sh (prod)/run_mds.sh~

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+export PYTHONPATH=$ROOT/src
+python $ROOT/src/cep/ib_mds.py $ROOT/src/config/app.cfg

sh (prod)/run_opt_serve.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=~/l1304/production/finopt
+export PYTHONPATH=$ROOT
+python $ROOT/finopt/opt_serve.py $ROOT/config/app.cfg

sh (prod)/run_opt_serve.sh~

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+export PYTHONPATH=$ROOT/src
+python $ROOT/src/finopt/opt_serve.py $ROOT/src/config/app.cfg

sh (prod)/run_options_data.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+#ROOT={replace-path}
+ROOT=~/l1304/production/finopt
+export PYTHONPATH=$ROOT
+python $ROOT/finopt/options_data.py $ROOT/config/app.cfg

sh (prod)/run_options_data.sh~

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+export PYTHONPATH=$ROOT/src
+python $ROOT/src/finopt/options_data.py $ROOT/src/config/app.cfg

sh (prod)/start-alert.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+python $FINOPT_HOME/comms/alert_bot.py $FINOPT_HOME/config/app.cfg
+

sh (prod)/stop-alert.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ps ax | grep -i 'alert' | grep python | grep -v grep | awk '{print $1}' | xargs kill -SIGTERM
+
+

sh (prod)/t1.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+ROOT=$FINOPT_HOME
+FINDATA=$ROOT/../data 
+SRC=$ROOT
+KAFKA_ASSEMBLY_JAR=$ROOT/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py $FINDATA/mds_files/large_up_1002
+

sh (prod)/t1.sh~

@@ -0,0 +1,11 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+

sh/greeks_changes.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+OPTIONS="--driver-memory 2g"
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit   $OPTIONS --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/greeks_changes.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+

sh/options_chain.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+python $FINOPT_HOME/finopt/options_chain.py $FINOPT_HOME/config/app.cfg
+

sh/portfoliokf.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=/home/larry/l1304/workspace/finopt/src
+export PYTHONPATH=$ROOT
+python $ROOT/finopt/portfolio_kf.py $ROOT/config/app.cfg

sh/sample_tws.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+python $FINOPT_HOME/comms/sample_tws_client.py 2

sh/sample_tws2.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+python $FINOPT_HOME/comms/sample_tws_client.py 2

sh/sample_tws3.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+python $FINOPT_HOME/comms/sample_tws_client.py 3

sh/start_twsgw.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+ROOT=$FINOPT_HOME
+export PYTHONPATH=$FINOPT_HOME:$PYTHONPATH
+# real time mode
+python $FINOPT_HOME/comms/tws_gateway.py $FINOPT_HOME/config/app.cfg

sh/stop_twsgw.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+ps ax | grep -i 'tws_gateway' | grep python | grep -v grep | awk '{print $1}' | xargs kill -SIGTERM
+
+