initial version

cep/generic_stream.py

@@ -0,0 +1,50 @@
+import sys
+
+from pyspark import SparkContext
+from pyspark.streaming import StreamingContext
+from pyspark.streaming.kafka import KafkaUtils
+#import optcal
+import json
+
+
+def process(time, rdd):
+        #print (time, rdd)
+        list =  (rdd.collect())
+        #print ('process---> %s' %list)
+        print '\n'.join ('%s %s'% (l['contract'], l['price']) for l in list) 
+
+def psize(time, rdd):
+        list = rdd.collect()
+        print '\n'.join ('%s:%s'% (l[0],l[1]) for l in list)
+        
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: ib_test02.py <broker_list ex: vsu-01:2181>")
+        exit(-1)
+
+    app_name = "IbMarketDataStream"
+    sc = SparkContext(appName= app_name)
+    ssc = StreamingContext(sc, 2)
+    ssc.checkpoint('./checkpoint')
+
+
+
+    brokers = sys.argv[1]
+    #kvs = KafkaUtils.createDirectStream(ssc, ['ib_tick_price', 'ib_tick_size'], {"metadata.broker.list": brokers})
+    kvs = KafkaUtils.createStream(ssc, brokers, app_name, {'ib_tick_price':1, 'ib_tick_size':1})
+
+    lines = kvs.map(lambda x: x[1])
+    msg = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == 1 and x['typeName']== 'tickPrice')).window(4, 2)
+    size = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == 1 and x['typeName']== 'tickSize'))\
+            .map(lambda x: (x['tickerId'], x['size'])).window(4,2) 
+    avg = size.reduceByKey(lambda a,b: (a+b)/2).window(4, 2)
+    
+
+#    mix.pprint()
+
+    msg.foreachRDD(process)
+    avg.foreachRDD(psize)
+    ssc.start()
+    ssc.awaitTermination()
+

cep/ib_mds.py

@@ -0,0 +1,313 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import ConfigParser
+from ib.ext.Contract import Contract
+from ib.opt import ibConnection, message
+from time import sleep
+import time, datetime
+from os import listdir
+from os.path import isfile, join
+from threading import Lock
+from comms.ib_heartbeat import IbHeartBeat
+import threading, urllib2
+
+#from options_data import ContractHelper
+
+import finopt.options_data as options_data
+from kafka.client import KafkaClient
+from kafka.producer import SimpleProducer
+from comms.alert_bot import AlertHelper
+
+## 
+## to run, start kafka server on vsu-01 <administrator, password>
+## start ibgw or tws
+## edit subscript list
+## check the settings in the console
+## 
+
+class IbKafkaProducer():
+    
+    config = None
+    con = None
+    quit = False
+    producer = None
+    IB_TICK_PRICE = None
+    IB_TICK_SIZE = None
+    toggle = False
+    persist = {}
+    ibh = None
+    tlock = None
+    ib_conn_status = None
+    
+    id2contract = {'conId': 1, 'id2contracts': {} }
+    
+    
+    def __init__(self, config):
+        
+        self.config = config
+        self.tlock = Lock()
+        host = self.config.get("ib_mds", "ib_mds.gateway").strip('"').strip("'")
+        port = int(self.config.get("ib_mds", "ib_mds.ib_port"))
+        appid = int(self.config.get("ib_mds", "ib_mds.appid.id"))  
+        kafka_host = self.config.get("cep", "kafka.host").strip('"').strip("'")
+        
+        self.persist['is_persist'] = self.config.get("ib_mds", "ib_mds.is_persist")
+        self.persist['persist_dir'] =self.config.get("ib_mds", "ib_mds.persist_dir").strip('"').strip("'")
+        self.persist['file_exist'] = False
+        self.persist['spill_over_limit'] = int(self.config.get("ib_mds", "ib_mds.spill_over_limit"))
+
+        
+        IbKafkaProducer.IB_TICK_PRICE = self.config.get("cep", "kafka.ib.topic.tick_price").strip('"').strip("'")
+        IbKafkaProducer.IB_TICK_SIZE = self.config.get("cep", "kafka.ib.topic.tick_size").strip('"').strip("'")
+        self.con = ibConnection(host, port, appid)
+        self.con.registerAll(self.on_ib_message)
+        rc = self.con.connect()
+        if rc:
+            self.ib_conn_status = 'OK'
+        logging.info('******* Starting IbKafkaProducer)
+        logging.info('IbKafkaProducer: connection status to IB is %d' % rc)
+        logging.info('IbKafkaProducer: connecting to kafka host: %s...' % kafka_host)
+        logging.info('IbKafkaProducer: message mode is async')
+        
+        client = KafkaClient(kafka_host)
+        self.producer = SimpleProducer(client, async=False)
+        
+        # start heart beat monitor
+        self.ibh = IbHeartBeat(config)
+        self.ibh.register_listener([self.on_ib_conn_broken])
+        self.ibh.run()        
+    
+    def pub_cn_index(self, sec):
+        
+        qs = '0000001,1399001,1399300'
+        url = 'http://api.money.126.net/data/feed/%s?callback=ne3587367b7387dc' % qs
+        
+        while 1:
+        
+            pg = urllib2.urlopen(url.encode('utf-8'))    
+            s = pg.read().replace('ne3587367b7387dc(', '')            
+            s = s[:len(s)-2]
+             
+            d = json.loads(s)
+            
+            tick_id = 9000
+            for k,v in d.iteritems():
+                
+                
+                c_name  = '%s-%s-%s-%s' % (v['type'],v['code'],0,0)
+
+                msg =  "%s,%d,%d,%s,%s" % (datetime.datetime.now().strftime('%Y%m%d%H%M%S'), tick_id ,\
+                                                                4, v['price'], c_name)
+                print msg
+                self.producer.send_messages('my.price', msg.encode('utf-8'))
+                tick_id+=1
+
+            sleep(sec)
+        
+        
+    def add_contract(self, tuple):
+        print tuple
+        c = options_data.ContractHelper.makeContract(tuple)
+        self.con.reqMktData(self.id2contract['conId'], c, '', False)
+        self.id2contract['id2contracts'][self.id2contract['conId']] = options_data.ContractHelper.makeRedisKeyEx(c)
+        self.id2contract['conId']+=1
+        
+    def on_ib_conn_broken(self, msg):
+        logging.error('IbKafkaProducer: connection is broken!')
+        self.ib_conn_status = 'ERROR'
+        self.tlock.acquire()
+        try:
+            if self.ib_conn_status == 'OK':
+                return
+            
+            self.con.eDisconnect()
+    
+            
+            host = self.config.get("ib_mds", "ib_mds.gateway").strip('"').strip("'")
+            port = int(self.config.get("ib_mds", "ib_mds.ib_port"))
+            appid = int(self.config.get("ib_mds", "ib_mds.appid.id"))        
+            self.con = ibConnection(host, port, appid)
+            self.con.registerAll(self.on_ib_message)
+            rc = None
+            while not rc:
+                logging.error('IbKafkaProducer: attempt reconnection!')
+                rc = self.con.connect()
+                logging.info('IbKafkaProducer: connection status to IB is %d (0-broken 1-good)' % rc)
+                sleep(2)
+            
+            # resubscribe tickers again!
+            self.load_tickers()                
+            a = AlertHelper(self.config)
+            a.post_msg('ib_mds recovered from broken ib conn, re-subscribe tickers...')            
+            logging.debug('on_ib_conn_broken: completed restoration. releasing lock...')
+            self.ib_conn_status = 'OK'
+            
+        finally:
+            self.tlock.release()          
+        
+
+        
+                
+    
+    def on_ib_message(self, msg):  
+
+
+        def create_tick_kmessage(msg):
+            d = {}
+            for t in msg.items():
+                d[t[0]] = t[1]
+            d['ts'] = time.time()
+            d['contract'] = self.id2contract['id2contracts'][msg.tickerId]
+            d['typeName'] = msg.typeName
+            d['source'] = 'IB'
+            return json.dumps(d)
+
+        
+        if msg.typeName in ['tickPrice', 'tickSize']:
+
+            t = create_tick_kmessage(msg)             
+            logging.debug(t)   
+            if self.toggle:
+                print t
+            self.producer.send_messages(IbKafkaProducer.IB_TICK_PRICE if msg.typeName == 'tickPrice' else IbKafkaProducer.IB_TICK_SIZE, t) 
+            if self.persist['is_persist']:
+                self.write_message_to_file(t)
+                
+            
+            
+#         print ",%0.4f,%0.4f" % (msg.pos, msg.avgCost)
+#         self.producer.send_messages('my.topic', "%0.4f,%0.4f" % (msg.pos, msg.avgCost))
+        elif msg.typeName in ['error']:
+            logging.error('on_ib_message: %s %s' % (msg.errorMsg, self.id2contract['id2contracts'][msg.id] if msg.errorCode == 200 else str(msg.errorCode)) )
+        else:
+           if self.toggle:
+               print msg
+                
+         
+    def write_message_to_file(self, t):
+        if self.persist['file_exist'] == False:
+            fn = '%s/ibkdump-%s.txt' % (self.persist['persist_dir'], datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
+            logging.debug('write_message_to_file: path %s', fn)
+            self.persist['fp'] = open(fn, 'w')
+            self.persist['file_exist'] = True
+            self.persist['spill_over_count'] = 0
+        
+        self.persist['fp'].write('%s|%s\n' % (datetime.datetime.now().strftime('%Y%m%d%H%M%S'), t))
+        self.persist['spill_over_count'] +=1
+        #print self.persist['spill_over_count']
+        if self.persist['spill_over_count'] == self.persist['spill_over_limit']:
+            self.persist['fp'].flush()
+            self.persist['fp'].close()
+            self.persist['file_exist'] = False
+
+    def load_tickers(self, path=None):
+        
+        self.id2contract = {'conId': 1, 'id2contracts': {} }
+        
+        
+        if path is None:
+            path = self.config.get("cep", "ib.subscription.fileloc").strip('"').strip("'")
+        logging.info('load_tickers: attempt to open file %s' % path)
+        fr = open(path)
+        for l in fr.readlines():
+            if l[0] <> '#':
+                 
+                self.add_contract(tuple([t for t in l.strip('\n').split(',')]))
+            
+
+    def do_work(self):
+        while not self.quit:
+            pass
+        
+    def run_forever(self):
+        t = threading.Thread(target = self.do_work, args=())
+        t.start()
+        self.console()
+        print 'pending ib connection to shut down...'
+        self.disconnect()
+        t.join()
+        print 'shutdown complete.'
+
+        
+    def disconnect(self):
+
+        self.con.disconnect()
+        if 'fp' in self.persist and self.persist['fp']:
+            self.persist['fp'].flush()
+            self.persist['fp'].close()
+        self.quit = True
+        self.ibh.shutdown()
+
+
+    def replay(self, dir_loc):
+        
+        def process_msg(fn):
+            fp = open(fn)
+            last_ts = None
+            for line in fp:
+                msg = line.split('|')[1]
+                msg_ts = json.loads(msg)['ts']
+#         
+#         files = [f if f.size() > 0 else None for f in dir_loc]
+        files = sorted([ f for f in listdir(dir_loc) if isfile(join(dir_loc,f)) ])            
+        for f in files:
+            process_msg(f)
+        
+
+    def console(self):
+        try:
+            while not self.quit:
+                print "Available commands are: l - list all subscribed contracts, a <symbol> <sectype> <exch> <ccy>"
+                print "                        t - turn/on off  output q - terminate program"
+                cmd = raw_input(">>")
+                input = cmd.split(' ')
+                if input[0]  == "q":
+                    print 'quit command received...'
+                    self.quit = True
+                elif input[0] == 'l' :
+                    print ''.join('%s: %s\n' % (k, v) for k, v in self.id2contract['id2contracts'].iteritems())
+                elif input[0] == 'a':
+                    self.add_contract((input[1],input[2],input[3],input[4],'',0,''))
+                elif input[0] == 't':
+                    self.toggle = False if self.toggle else True
+                else:
+                    pass
+            
+        except:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
+            
+
+if __name__ == '__main__':
+           
+#     logging.basicConfig(#filename = "log/port.log", filemode = 'w', 
+#                         level=logging.INFO,
+#                         format='%(asctime)s %(levelname)-8s %(message)s')      
+#     
+#     config = ConfigParser.ConfigParser()
+#     config.read("../config/app.cfg")
+#     ik = IbKafkaProducer(config)
+#     ik.load_tickers()    
+#     ik.run_forever()
+    
+    
+    
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0:      
+        raise ValueError, "Failed to open config file" 
+    
+      
+    logconfig = eval(config.get("ib_mds", "ib_mds.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)    
+    ik = IbKafkaProducer(config)
+    ik.load_tickers()    
+    ik.run_forever()

cep/momentum.py

@@ -0,0 +1,201 @@
+import sys
+
+from pyspark import SparkContext
+from pyspark.streaming import StreamingContext
+from pyspark.streaming.kafka import KafkaUtils
+from numpy import *
+import pylab
+from scipy import stats
+
+import threading
+import time
+import os
+##
+##
+##
+## This example demonstrates the use of accumulators and broadcast 
+## and how to terminate spark running jobs
+##
+##
+
+##
+##
+## insert the path so spark-submit knows where
+## to look for a file located in a given directory
+##
+## the other method is to export PYTHONPATH before 
+## calling spark-submit
+##
+# import sys
+# sys.path.insert(0, '/home/larry-13.04/workspace/finopt/cep')
+print sys.path
+
+
+#import optcal
+import json
+import numpy
+#from finopt.cep.redisQueue import RedisQueue
+from comms.redisQueue import RedisQueue
+
+
+
+def persist(time, rdd):
+        #print (time, rdd)
+        #lt =  (rdd.collect())
+        
+        rdd.saveAsTextFile("./rdd/rdd-%s-%03d" % (Q.value['rname'], NumProcessed.value))
+        print 'process................... %d' % NumProcessed.value
+        NumProcessed.add(1)
+        #pass
+        #print '\n'.join ('%d %s'% (l[0], ''.join(('%f'% e) for e in l[1])) for l in list) 
+ 
+def simple(time, rdd):
+
+        
+        lt =  (rdd.collect())
+        
+        if lt:
+            
+            first = lt[0][1][0]
+            last_pos = len(lt) - 1
+            last = lt[last_pos][1][0]
+            change = (last - first) / last
+            print change, first, last, len(lt)
+            print 'process................... %d' % NumProcessed.value
+        NumProcessed.add(1)
+       
+
+def cal_trend(time, rdd):
+
+
+    def detect_trend(x1, y1, num_points_ahead, ric):
+        z4 = polyfit(x1, y1, 3) 
+        p4 = poly1d(z4) # construct the polynomial 
+        #print y1
+        
+        z5 = polyfit(x1, y1, 4)
+        p5 = poly1d(z5)
+        
+        extrap_y_max_limit = len(x1) * 2  # 360 days
+        x2 = linspace(0, extrap_y_max_limit, 50) # 0, 160, 100 means 0 - 160 with 100 data points in between
+        pylab.switch_backend('agg') # switch to agg backend that support writing in non-main threads
+        pylab.plot(x1, y1, 'o', x2, p4(x2),'-g', x2, p5(x2),'-b')
+        pylab.legend(['%s to fit' % ric, '4th degree poly', '5th degree poly'])
+        #pylab.axis([0,160,0,10])
+#         
+        pylab.axis([0,len(x1)*1.1, min(y1)*0.997,max(y1)*1.002])  # first pair tells the x axis boundary, 2nd pair y axis boundary 
+        
+        # compute the slopes of each set of data points
+        # sr - slope real contains the slope computed from real data points from d to d+5 days
+        # s4 - slope extrapolated by applying 4th degree polynomial
+        y_arraysize = len(y1)
+#         s4, intercept, r_value, p_value, std_err = stats.linregress(range(0,num_points_ahead), [p4(i) for i in range(y_arraysize,y_arraysize + num_points_ahead )])
+#         s5, intercept, r_value, p_value, std_err = stats.linregress(range(0,num_points_ahead), [p5(i) for i in range(y_arraysize,y_arraysize + num_points_ahead )])
+        s4, intercept, r_value, p_value, std_err = stats.linregress(x1, y1)
+        s5, intercept, r_value, p_value, std_err = stats.linregress(x1, y1)
+        
+          
+        rc = (1.0 if s4 > 0.0 else 0.0, 1.0 if s5 > 0.0 else 0.0)
+        print s4, s5, rc, y_arraysize
+        #pylab.show()
+        pylab.savefig('../data/extrapolation/%s-%s.png' % (ric, time))
+
+        d = Q.value
+        q = RedisQueue(d['qname'], d['namespace'], d['host'], d['port'], d['db'])
+        q.put((time, y1))
+
+#         # clear memory
+        pylab.close()
+        return rc
+
+    ls = rdd.collect()  
+#     print [y[1][0] for y in ls]
+#     print len(ls), [range(len(ls))]
+#     print len([y[1][0] for y in ls])
+    if ls:
+        rc = detect_trend(range(len(ls)), [y[1][0] for y in ls], 5, '_HSI')
+    
+
+# to run from command prompt
+# 0. start kafka broker
+# 1. edit subscription.txt and prepare 2 stocks
+# 2. run ib_mds.py 
+# 3. spark-submit  --jars spark-streaming-kafka-assembly_2.10-1.4.1.jar ./alerts/pairs_corr.py vsu-01:2181 
+
+# http://stackoverflow.com/questions/3425439/why-does-corrcoef-return-a-matrix
+# 
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print("Usage: %s <broker_list ex: vsu-01:2181>  <rdd_name> <tick id> <fn name>" % sys.argv[0])
+        print("Usage: to gracefully shutdown type echo 1 > /tmp/flag at the terminal")
+        exit(-1)
+
+    app_name = "Momentum"
+    sc = SparkContext(appName= app_name) #, pyFiles = ['./cep/redisQueue.py'])
+    ssc = StreamingContext(sc, 2)
+    ssc.checkpoint('../checkpoint')
+
+    
+
+    brokers, qname, id, fn  = sys.argv[1:]
+    id = int(id)
+    
+    #
+    # demonstrate how to use broadcast variable
+    #
+    NumProcessed = sc.accumulator(0)
+    Q = sc.broadcast({'rname': 'rname', 'qname': qname, 'namespace': 'mdq', 'host': 'localhost', 'port':6379, 'db': 3})
+    
+    #kvs = KafkaUtils.createDirectStream(ssc, ['ib_tick_price', 'ib_tick_size'], {"metadata.broker.list": brokers})
+    kvs = KafkaUtils.createStream(ssc, brokers, app_name, {'ib_tick_price':1, 'ib_tick_size':1})
+
+    lines = kvs.map(lambda x: x[1])
+    s1 = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == id and x['typeName']== 'tickPrice'))\
+                .filter(lambda x: (x['field'] == 1))\
+                .map(lambda x: (id, x['price'])).window(30, 20)
+                
+    s2 = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == id and x['typeName']== 'tickSize'))\
+                .filter(lambda x: (x['field'] == 5))\
+                .map(lambda x: (id, x['size'])).window(30, 20)
+                
+    
+    
+                
+    #trades = s1.join(s2)
+    trades = s1.leftOuterJoin(s2)
+    
+
+    #s1.pprint()
+    trades.pprint()
+    trades.foreachRDD(eval(fn))
+    
+        
+    def do_work():
+
+        while 1:
+            # program will stop after processing 40 rdds
+            if NumProcessed.value == 999:
+                break            
+            # program will stop on detecting a 1 in the flag file
+            try:
+                f = open('/tmp/flag')
+                l = f.readlines()
+                print 'reading %s' % l[0]
+                if '1' in l[0]:
+                    break
+                f.close()
+                time.sleep(2)
+            except:
+                continue
+            
+        print 'terminating...'        
+        ssc.stop(True, False) 
+        os.remove('/tmp/flag')           
+            
+        
+    t = threading.Thread(target = do_work, args=())
+    t.start()
+    ssc.start()
+    ssc.awaitTermination()
+

cep/momentum2.py

@@ -0,0 +1,233 @@
+import sys
+
+from pyspark import SparkContext
+from pyspark.streaming import StreamingContext
+from pyspark.streaming.kafka import KafkaUtils
+from numpy import *
+import pylab
+from scipy import stats
+
+import threading
+import time
+import os
+##
+##
+##
+## This example demonstrates the use of accumulators and broadcast 
+## and how to terminate spark running jobs
+## 
+## it also demonstrates how to send alerts via xmpp
+## (requires prosody server running and redisQueue)
+##
+##
+
+##
+##
+## insert the path so spark-submit knows where
+## to look for a file located in a given directory
+##
+## the other method is to export PYTHONPATH before 
+## calling spark-submit
+##
+# import sys
+# sys.path.insert(0, '/home/larry-13.04/workspace/finopt/cep')
+print sys.path
+
+
+#import optcal
+import json
+import numpy
+#from finopt.cep.redisQueue import RedisQueue
+from comms.redisQueue import RedisQueue
+
+
+
+def f1(time, rdd):
+    lt =  rdd.collect()
+    for l in lt:
+        print ''.join('%s {%s}' % (l[0], ','.join('%f'% e for e in l[1])))
+        xx = [e for e in l[1]]
+        print xx
+        if len(xx) > 1:
+            first = xx[0]
+            last_pos = len(xx) - 1
+            last = xx[last_pos]
+            change = (last - first) / last   
+            print '%f' % change
+    #rint '\n'.join ('%s %s'% (l[0], ''.join(('%f'% e) for e in l[1])) for l in lt)
+
+def persist(time, rdd):
+        #print (time, rdd)
+        #lt =  (rdd.collect())
+        
+        rdd.saveAsTextFile("./rdd/rdd-%s-%03d" % (Q.value['rname'], NumProcessed.value))
+        print 'process................... %d' % NumProcessed.value
+        NumProcessed.add(1)
+        #pass
+        #print '\n'.join ('%d %s'% (l[0], ''.join(('%f'% e) for e in l[1])) for l in list) 
+ 
+def simple(time, rdd):
+        lt =  (rdd.collect())
+        
+        if lt:
+            
+            first = lt[0][1][0]
+            last_pos = len(lt) - 1
+            last = lt[last_pos][1][0]
+            change = (last - first) / last
+            msg = '%0.6f, %0.2f, %0.2f, %0.2f' % (change, first, last, last_pos)
+            print 'process............. %d {%s}' % (NumProcessed.value, msg)
+            if abs(change) > Threshold.value:
+                d = Q.value
+                q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'], d['port'], d['db'])
+                q.put(msg)
+            
+            
+        NumProcessed.add(1)
+        #print (time, rdd)
+
+       
+        
+       
+
+def cal_trend(time, rdd):
+
+
+    def detect_trend(x1, y1, num_points_ahead, ric):
+        z4 = polyfit(x1, y1, 3) 
+        p4 = poly1d(z4) # construct the polynomial 
+        #print y1
+        
+        z5 = polyfit(x1, y1, 4)
+        p5 = poly1d(z5)
+        
+        extrap_y_max_limit = len(x1) * 2  # 360 days
+        x2 = linspace(0, extrap_y_max_limit, 50) # 0, 160, 100 means 0 - 160 with 100 data points in between
+        pylab.switch_backend('agg') # switch to agg backend that support writing in non-main threads
+        pylab.plot(x1, y1, 'o', x2, p4(x2),'-g', x2, p5(x2),'-b')
+        pylab.legend(['%s to fit' % ric, '4th degree poly', '5th degree poly'])
+        #pylab.axis([0,160,0,10])
+#         
+        pylab.axis([0,len(x1)*1.1, min(y1)*0.997,max(y1)*1.002])  # first pair tells the x axis boundary, 2nd pair y axis boundary 
+        
+        # compute the slopes of each set of data points
+        # sr - slope real contains the slope computed from real data points from d to d+5 days
+        # s4 - slope extrapolated by applying 4th degree polynomial
+        y_arraysize = len(y1)
+#         s4, intercept, r_value, p_value, std_err = stats.linregress(range(0,num_points_ahead), [p4(i) for i in range(y_arraysize,y_arraysize + num_points_ahead )])
+#         s5, intercept, r_value, p_value, std_err = stats.linregress(range(0,num_points_ahead), [p5(i) for i in range(y_arraysize,y_arraysize + num_points_ahead )])
+        s4, intercept, r_value, p_value, std_err = stats.linregress(x1, y1)
+        s5, intercept, r_value, p_value, std_err = stats.linregress(x1, y1)
+        
+          
+        rc = (1.0 if s4 > 0.0 else 0.0, 1.0 if s5 > 0.0 else 0.0)
+        print s4, s5, rc, y_arraysize
+        #pylab.show()
+        pylab.savefig('../data/extrapolation/%s-%s.png' % (ric, time))
+        pylab.close()
+
+        d = Q.value
+        q = RedisQueue(d['qname'], d['namespace'], d['host'], d['port'], d['db'])
+        q.put((time, y1, s4, s5))
+
+        if abs(s4) > Threshold.value:
+            d = Q.value
+            q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'], d['port'], d['db'])
+            q.put('%s %0.2f %0.2f' % (ric, s4, s5))
+
+        return rc
+
+    ls = rdd.collect()  
+#     print [y[1][0] for y in ls]
+#     print len(ls), [range(len(ls))]
+#     print len([y[1][0] for y in ls])
+    if ls:
+        rc = detect_trend(range(len(ls)), [y[1][0] for y in ls], 5, '_HSI')
+    
+
+# to run from command prompt
+# 0. start kafka broker
+# 1. edit subscription.txt and prepare 2 stocks
+# 2. run ib_mds.py 
+# 3. spark-submit  --jars spark-streaming-kafka-assembly_2.10-1.4.1.jar ./alerts/pairs_corr.py vsu-01:2181 
+
+# http://stackoverflow.com/questions/3425439/why-does-corrcoef-return-a-matrix
+# 
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print("Usage: %s <broker_list ex: vsu-01:2181>  <rdd_name> <tick id> <fn name>" % sys.argv[0])
+        print("Usage: to gracefully shutdown type echo 1 > /tmp/flag at the terminal")
+        exit(-1)
+
+    app_name = "Momentum"
+    sc = SparkContext(appName= app_name) #, pyFiles = ['./cep/redisQueue.py'])
+    ssc = StreamingContext(sc, 2)
+    ssc.checkpoint('../checkpoint')
+
+    
+
+    brokers, qname, id, fn  = sys.argv[1:]
+    id = int(id)
+    
+    #
+    # demonstrate how to use broadcast variable
+    #
+    NumProcessed = sc.accumulator(0)
+    Q = sc.broadcast({'rname': 'rname', 'qname': qname, 'namespace': 'mdq', 'host': 'localhost', 'port':6379, 'db': 3, 'alert_bot_q': ('msg_bot', 'chatq')})
+    Threshold = sc.broadcast(0.00015)
+    #kvs = KafkaUtils.createDirectStream(ssc, ['ib_tick_price', 'ib_tick_size'], {"metadata.broker.list": brokers})
+    kvs = KafkaUtils.createStream(ssc, brokers, app_name, {'ib_tick_price':1, 'ib_tick_size':1})
+
+    lines = kvs.map(lambda x: x[1])
+#     s1 = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == id and x['typeName']== 'tickPrice'))\
+#                 .filter(lambda x: (x['field'] == 4))\
+#                 .map(lambda x: (id, x['price'])).window(60, 30)
+#                 
+#     s2 = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == id and x['typeName']== 'tickSize'))\
+#                 .filter(lambda x: (x['field'] == 5))\
+#                 .map(lambda x: (id, x['size'])).window(60, 30)
+#                 
+#     
+#     
+#                 
+#     trades = s1.join(s2)
+    s1 = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] in [1,2] and x['typeName']== 'tickPrice'))\
+                .filter(lambda x: (x['field'] == 4))\
+                .map(lambda x: (x['tickerId'], x['price'])).reduceByKey(lambda x,y: (x+y)/2).groupByKeyAndWindow(30, 20, 1)    
+    
+    
+    s1.foreachRDD(f1)
+    #trades.pprint()
+    #trades.foreachRDD(eval(fn))
+    
+        
+    def do_work():
+
+        while 1:
+            # program will stop after processing 40 rdds
+#             if NumProcessed.value == 70:
+#                 break            
+            # program will stop on detecting a 1 in the flag file
+            try:
+                f = open('/tmp/flag')
+                l = f.readlines()
+                print 'reading %s' % l[0]
+                if '1' in l[0]:
+                    break
+                f.close()
+                time.sleep(2)
+            except IOError:
+                continue
+            
+        print 'terminating..........'        
+        ssc.stop(True, False) 
+        os.remove('/tmp/flag') 
+        sys.exit()          
+            
+        
+    t = threading.Thread(target = do_work, args=())
+    t.start()
+    ssc.start()
+    ssc.awaitTermination()
+

cep/pairs_corr.py

@@ -0,0 +1,76 @@
+import sys
+
+from pyspark import SparkContext
+from pyspark.streaming import StreamingContext
+from pyspark.streaming.kafka import KafkaUtils
+#import optcal
+import json
+import numpy
+
+
+
+def process(time, rdd):
+        #print (time, rdd)
+        lt =  (rdd.collect())
+        #print '\n'.join ('%d %s'% (l[0], ''.join(('%f'% e) for e in l[1])) for l in list) 
+        if len(lt) == 2:
+            a = list(lt[0][1])
+            b = list(lt[1][1])
+            #print a, b
+            corr = 0.0
+            if len(a) > 1 and len(b) > 1:                
+                if len(a) > len(b):
+                    corr= numpy.corrcoef(a[:len(b)], b)
+                else:
+                    corr= numpy.corrcoef(b[:len(a)], a)
+                    
+                print "%s corr---> %f" % (time.strftime('%Y%m%d %H:%M:%S'), corr.tolist()[0][1])
+            #print numpy.corrcoef(list(lt[0][1]), list(lt[1][1]))
+
+
+       
+
+# to run from command prompt
+# 0. start kafka broker
+# 1. edit subscription.txt and prepare 2 stocks
+# 2. run ib_mds.py 
+# 3. spark-submit  --jars spark-streaming-kafka-assembly_2.10-1.4.1.jar ./alerts/pairs_corr.py vsu-01:2181 
+
+# http://stackoverflow.com/questions/3425439/why-does-corrcoef-return-a-matrix
+# 
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: ib_test02.py <broker_list ex: vsu-01:2181>")
+        exit(-1)
+
+    app_name = "IbMarketDataStream"
+    sc = SparkContext(appName= app_name)
+    ssc = StreamingContext(sc, 2)
+    ssc.checkpoint('./checkpoint')
+
+
+
+    brokers = sys.argv[1]
+    #kvs = KafkaUtils.createDirectStream(ssc, ['ib_tick_price', 'ib_tick_size'], {"metadata.broker.list": brokers})
+    kvs = KafkaUtils.createStream(ssc, brokers, app_name, {'ib_tick_price':1, 'ib_tick_size':1})
+
+    lines = kvs.map(lambda x: x[1])
+    uso = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == 1 and x['typeName']== 'tickPrice'))\
+                .map(lambda x: (1, x['price'])).window(8, 6)
+    dug = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == 2 and x['typeName']== 'tickPrice'))\
+                .map(lambda x: (2, x['price'])).window(8, 6)
+                
+                
+    pair = uso.union(dug).groupByKey()
+    # sample values are empty, one element, and 2 elements
+    #(1, <pyspark.resultiterable.ResultIterable object at 0x7fae53a187d0>)
+    #(2, <pyspark.resultiterable.ResultIterable object at 0x7fae53a18c50>)
+
+    
+    
+    #pair.pprint()
+    pair.foreachRDD(process)
+    ssc.start()
+    ssc.awaitTermination()
+

cep/pairs_corr_redis.py

@@ -0,0 +1,108 @@
+import sys
+
+from pyspark import SparkContext
+from pyspark.streaming import StreamingContext
+from pyspark.streaming.kafka import KafkaUtils
+
+
+##
+##
+## insert the path so spark-submit knows where
+## to look for a file located in a given directory
+##
+## the other method is to export PYTHONPATH before 
+## calling spark-submit
+##
+# import sys
+# sys.path.insert(0, '/home/larry-13.04/workspace/finopt/cep')
+print sys.path
+
+
+#import optcal
+import json
+import numpy
+#from finopt.cep.redisQueue import RedisQueue
+from comms.redisQueue import RedisQueue
+
+
+
+def process(time, rdd):
+        #print (time, rdd)
+        lt =  (rdd.collect())
+        #print '\n'.join ('%d %s'% (l[0], ''.join(('%f'% e) for e in l[1])) for l in list) 
+        if len(lt) == 2:
+            a = list(lt[0][1])
+            b = list(lt[1][1])
+            #print a, b
+            corr = 0.0
+            if len(a) > 1 and len(b) > 1:                
+                if len(a) > len(b):
+                    corr= numpy.corrcoef(a[:len(b)], b)
+                else:
+                    corr= numpy.corrcoef(b[:len(a)], a)
+                    
+                print "%s corr---> %f" % (time.strftime('%Y%m%d %H:%M:%S'), corr.tolist()[0][1])
+                d = Q.value
+
+                q = RedisQueue(d['qname'], d['namespace'], d['host'], d['port'], d['db'])
+                corr = corr.tolist()[0][1]
+                if not numpy.isnan(corr):
+                    print 'insert into redis'
+                    q.put((time,  corr))
+            #print numpy.corrcoef(list(lt[0][1]), list(lt[1][1]))
+
+
+       
+
+# to run from command prompt
+# 0. start kafka broker
+# 1. edit subscription.txt and prepare 2 stocks
+# 2. run ib_mds.py 
+# 3. spark-submit  --jars spark-streaming-kafka-assembly_2.10-1.4.1.jar ./alerts/pairs_corr.py vsu-01:2181 
+
+# http://stackoverflow.com/questions/3425439/why-does-corrcoef-return-a-matrix
+# 
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: pairs_corr_redis.py <broker_list ex: vsu-01:2181> <queue_name - for saving the correlations series>")
+        exit(-1)
+
+    app_name = "IbMarketDataStream"
+    sc = SparkContext(appName= app_name) #, pyFiles = ['./cep/redisQueue.py'])
+    ssc = StreamingContext(sc, 2)
+    ssc.checkpoint('./checkpoint')
+
+    
+
+    brokers, qname = sys.argv[1:]
+
+    
+    #
+    # demonstrate how to use broadcast variable
+    #
+    
+    Q = sc.broadcast({'qname': qname, 'namespace': 'mdq', 'host': 'localhost', 'port':6379, 'db': 3})
+    
+    #kvs = KafkaUtils.createDirectStream(ssc, ['ib_tick_price', 'ib_tick_size'], {"metadata.broker.list": brokers})
+    kvs = KafkaUtils.createStream(ssc, brokers, app_name, {'ib_tick_price':1, 'ib_tick_size':1})
+
+    lines = kvs.map(lambda x: x[1])
+    uso = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == 1 and x['typeName']== 'tickPrice'))\
+                .map(lambda x: (1, x['price'])).window(40, 30)
+    dug = lines.map(lambda line: json.loads(line)).filter(lambda x: (x['tickerId'] == 2 and x['typeName']== 'tickPrice'))\
+                .map(lambda x: (2, x['price'])).window(40, 30)
+                
+                
+    pair = uso.union(dug).groupByKey()
+    # sample values are empty, one element, and 2 elements
+    #(1, <pyspark.resultiterable.ResultIterable object at 0x7fae53a187d0>)
+    #(2, <pyspark.resultiterable.ResultIterable object at 0x7fae53a18c50>)
+
+    
+    
+    #pair.pprint()
+    pair.foreachRDD(process)
+    ssc.start()
+    ssc.awaitTermination()
+

comms/alert_bot.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    SleekXMPP: The Sleek XMPP Library
+    Copyright (C) 2010  Nathanael C. Fritz
+    This file is part of SleekXMPP.
+    See the file LICENSE for copying permission.
+"""
+import sys, traceback
+import json
+import logging
+import ConfigParser
+from time import sleep
+import time, datetime
+import sleekxmpp
+import redis
+from threading import Lock
+from redisQueue import RedisQueue
+
+import threading
+
+# Python versions before 3.0 do not use UTF-8 encoding
+# by default. To ensure that Unicode is handled properly
+# throughout SleekXMPP, we will set the default encoding
+# ourselves to UTF-8.
+if sys.version_info < (3, 0):
+    from sleekxmpp.util.misc_ops import setdefaultencoding
+    setdefaultencoding('utf8')
+else:
+    raw_input = input
+
+
+
+
+class AlertMsgBot(sleekxmpp.ClientXMPP):
+
+    """
+    A basic SleekXMPP bot that will log in, send a message,
+    and then log out.
+    """
+    config = None
+    rsq= None
+
+    recipients = None
+    quit = False
+    tlock = None
+    
+
+    def __init__(self, config):
+        
+
+        self.config = config
+        host = config.get("redis", "redis.server").strip('"').strip("'")
+        port = config.get("redis", "redis.port")
+        db = config.get("redis", "redis.db")
+        qname = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
+        qprefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
+        
+        self.rsq = RedisQueue(qname, qprefix, host, port, db)
+        logging.info('Connect to redis on server->%s:%s db->%s  qname->%s:%s' % (host, port, db, qprefix, qname))
+        jid = config.get("alert_bot", "msg_bot.jid").strip('"').strip("'")
+        password = config.get("alert_bot", "msg_bot.pass").strip('"').strip("'")
+        sleekxmpp.ClientXMPP.__init__(self, jid, password)
+        
+        self.recipients = eval(config.get("alert_bot", "msg_bot.recipients").strip('"').strip("'"))
+        self.tlock = Lock()
+
+        # The session_start event will be triggered when
+        # the bot establishes its connection with the server
+        # and the XML streams are ready for use. We want to
+        # listen for this event so that we we can initialize
+        # our roster.
+        self.add_event_handler("session_start", self.start, threaded=True)
+
+    def start(self, event):
+        """
+        Process the session_start event.
+        Typical actions for the session_start event are
+        requesting the roster and broadcasting an initial
+        presence stanza.
+        Arguments:
+            event -- An empty dictionary. The session_start
+                     event does not provide any additional
+                     data.
+        """
+        self.send_presence()
+        self.get_roster()
+
+        self.rsq.put("Greetings! The alert bot has just started!")
+        t = threading.Thread(target = self.process_alerts(), args=())
+        t.start()
+#         self.send_message(mto=self.recipient,
+#                           mbody=self.msg,
+#                           mtype='chat')
+
+        
+        
+        
+    def process_alerts(self):
+        self.quit = False
+        while self.quit <> True:
+
+            if not self.rsq.empty():
+                msg = self.rsq.get()
+                logging.debug('process_alerts: received msg: {%s}' % msg)
+                self.send_msg(msg)
+            sleep(1)
+        # Using wait=True ensures that the send queue will be
+        # emptied before ending the session.
+        self.disconnect(wait=True)
+
+
+    def send_msg(self, msg):
+        self.tlock.acquire()
+        try:
+            for r in self.recipients:
+                self.send_message(r, msg, mtype='chat')            
+        finally:
+            self.tlock.release()  
+        
+#
+# alert helper doesn't care whether the xmpp server 
+# has started or not, all it cares is being able to put the message 
+# into redisQueue
+class AlertHelper():
+    q = None
+    
+    def __init__(self, config):
+        rhost = config.get("redis", "redis.server").strip('"').strip("'")
+        rport = config.get("redis", "redis.port")
+        rdb = config.get("redis", "redis.db")
+        chatq = config.get("alert_bot", "msg_bot.redis_mq").strip('"').strip("'")
+        prefix = config.get("alert_bot", "msg_bot.redis_prefix").strip('"').strip("'")
+        
+        self.q = RedisQueue(chatq, prefix, rhost, rport, rdb)
+        
+    def post_msg(self, msg):
+        self.q.put(msg)      
+        
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0: 
+        raise ValueError, "Failed to open config file" 
+      
+    logconfig = eval(config.get("alert_bot", "msg_bot.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'
+    logging.basicConfig(**logconfig)
+
+    xmpp = AlertMsgBot(config)
+    xmpp.register_plugin('xep_0030') # Service Discovery
+    xmpp.register_plugin('xep_0199') # XMPP Ping
+
+    if xmpp.connect(): #('192.168.1.1', 5222), True, True, False):
+        xmpp.process(block=False)
+        logging.info('Complete initialization...Bot will now run forever')
+        a = AlertHelper(config)
+        a.post_msg('from AlertHelper: testing 123')
+    else:
+        print("Unable to connect.")

comms/ib_heartbeat.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import ConfigParser
+from ib.ext.Contract import Contract
+from ib.opt import ibConnection, message
+from time import sleep
+import time, datetime
+from os import listdir
+from os.path import isfile, join
+from comms.alert_bot import AlertHelper
+
+import threading, urllib2
+
+
+class IbHeartBeat():
+    config = None
+    quit = False
+    prev_state = ''
+    q = None
+    chat_handle = None
+    last_broken_time = None
+    alert_callbacks = []
+    
+    def __init__(self, config):
+        self.config = config
+        self.chat_handle = AlertHelper(config)              
+        
+        # ensure the message will get printed right away when the connection is first broken
+        self.last_broken_time =  datetime.datetime.now() - datetime.timedelta(seconds=90)  
+        
+        
+
+    def register_listener(self, fns):
+        self.alert_callbacks = [fn for fn in fns]
+        
+    def alert_listeners(self, msg):
+        [fn(msg) for fn in self.alert_callbacks]
+
+    def run(self):
+        t = threading.Thread(target = self.keep_trying, args=())
+        t.start()
+
+    def shutdown(self):
+        self.quit = True
+        
+    def keep_trying(self):
+        host = self.config.get("ib_heartbeat", "ib_heartbeat.gateway").strip('"').strip("'")
+        port = int(self.config.get("ib_heartbeat", "ib_heartbeat.ib_port"))
+        appid = int(self.config.get("ib_heartbeat", "ib_heartbeat.appid.id"))      
+        try_interval = int(self.config.get("ib_heartbeat", "ib_heartbeat.try_interval"))
+        suppress_msg_interval = int(self.config.get("ib_heartbeat", "ib_heartbeat.suppress_msg_interval"))
+        logging.info('ib gateway->%s:%d, appid->%d, try_interval->%d, suppress msg interval->%d' % \
+                     (host, port, appid, try_interval, suppress_msg_interval))
+        while not self.quit:
+            con = ibConnection(host, port, appid)
+            rc = con.connect()
+            if rc:
+                if self.prev_state == 'broken':
+                    msg = '*** Connection restored at %s **********' % datetime.datetime.now().strftime('%H:%M:%S')
+                    self.chat_handle.post_msg(msg)
+                    self.alert_listeners(msg)
+                    self.prev_state = ''
+                    # reset to a much earlier time
+                    self.last_broken_time = datetime.datetime.now() - datetime.timedelta(seconds=90)
+                con.eDisconnect()
+            else:
+                msg = '*** Connection to IB API is broken **********'
+                now = datetime.datetime.now()
+                self.prev_state = 'broken' 
+                #print now, self.last_broken_time, (now - self.last_broken_time).seconds
+                if (now - self.last_broken_time).seconds > suppress_msg_interval:
+                    self.chat_handle.post_msg(msg)
+                    self.alert_listeners(msg)
+                    self.last_broken_time = now 
+                    logging.error(msg)
+                
+            sleep(try_interval)
+            
+
+
+
+
+if __name__ == '__main__':
+           
+  
+    
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0:      
+        raise ValueError, "Failed to open config file" 
+    
+      
+    logconfig = eval(config.get("ib_mds", "ib_mds.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)    
+    ibh = IbHeartBeat(config)
+    
+    def warn_me(msg):
+        print 'warn_me: received %s' % msg
+    
+    ibh.register_listener([warn_me])
+    ibh.run()
+    
+    
+    

comms/redisQueue.py

@@ -0,0 +1,50 @@
+import redis
+import sys
+
+class RedisQueue(object):
+    """Simple Queue with Redis Backend"""
+    def __init__(self, name, namespace, host, port, db):
+        self.__db= redis.Redis(host, port, db)
+        self.key = '%s:%s' %(namespace, name)
+        
+        try:
+            self.__db.client_list()
+        except redis.ConnectionError:
+            print "RedisQueue failed to connect to the server. "
+        
+        #print host, port, db, self.key
+
+    def qsize(self):
+        """Return the approximate size of the queue."""
+        return self.__db.llen(self.key)
+
+    def empty(self):
+        """Return True if the queue is empty, False otherwise."""
+        return self.qsize() == 0
+
+    def put(self, item):
+        """Put item into the queue."""
+        self.__db.rpush(self.key, item)
+
+    def peek(self, pos=0):
+        return self.__db.lindex(self.key, pos)
+
+    def get(self, block=True, timeout=None):
+        """Remove and return an item from the queue. 
+
+        If optional args block is true and timeout is None (the default), block
+        if necessary until an item is available."""
+        if block:
+            item = self.__db.blpop(self.key, timeout=timeout)
+        else:
+            item = self.__db.lpop(self.key)
+
+        if item:
+            item = item[1]
+        return item
+
+    def get_nowait(self):
+        """Equivalent to get(False)."""
+        return self.get(False)
+    
+    

finopt/opt_serve.py

@@ -0,0 +1,401 @@
+# -*- coding: utf-8 -*-
+import sys, traceback
+import logging
+import os
+import ast
+import urllib, urllib2, cookielib
+import datetime
+import re
+import json
+import cherrypy
+import hashlib
+import uuid
+import redis
+import json
+import optcal
+import ConfigParser
+import portfolio
+
+
+class QServer(object):
+    
+    config = None
+    r_conn = None
+    
+    
+    def __init__(self, r_conn, config):
+        super(QServer, self).__init__()
+        QServer.r_conn = r_conn
+        QServer.config = config
+        #print QServer.r_conn
+        
+
+    
+    
+    @cherrypy.expose
+    def index(self):
+        
+        s_line = 'welcome!'
+        #r_host = cherrypy.request.app.config['redis']['redis.server']
+        #r_port = cherrypy.request.app.config['redis']['redis.port']
+        #r_db = cherrypy.request.app.config['redis']['redis.db']
+        #r_sleep = cherrypy.request.app.config['redis']['redis.sleep']
+    
+        #rs = redis.Redis(r_host, r_port, r_db)
+        rs = QServer.r_conn
+        s_line = rs.info()
+        html =''
+        for k, v in cherrypy.request.app.config.iteritems():
+            html = html + '<dt>%s</dt><dd>%s</dd>' % (k, v)
+       
+        impl_link = "<a href='./opt_implv'>options implied vol curves</a>"
+        pos_link = "<a href=./ws_position_chart>Positions</a>" 
+        return """<html><body><li>%s</li><li>%s</li><br><dl>%s</dl></br>%s</body></html>""" % (impl_link, pos_link, html, s_line)
+ 
+ 
+    @cherrypy.expose
+    def opt_chains(self): 
+        r_host = cherrypy.request.app.config['redis']['redis.server']
+        r_port = cherrypy.request.app.config['redis']['redis.port']
+        r_db = cherrypy.request.app.config['redis']['redis.db']
+        r_sleep = cherrypy.request.app.config['redis']['redis.sleep']
+        opt_chains = cherrypy.request.app.config['redis']['redis.datastore.key.option_chains']
+        rs = redis.Redis(r_host, r_port, r_db)        
+        opt_chain_tmpl = '%s%s/opt-chains-tmpl.html' % (cherrypy.request.app.config['/']['tools.staticdir.root'], cherrypy.request.app.config['/static']['tools.staticdir.tmpl'])
+ 
+        f = open(opt_chain_tmpl)
+        html_tmpl = f.read()
+
+        s_dict = rs.get(opt_chains)
+        matrix = json.loads(s_dict)
+        
+        strike = matrix.keys()[0]
+        print matrix
+        num_months = len(matrix[strike])
+        s = '["strike",'
+#         for i in range(num_months):          
+#             s = s + "'P-%s', 'C-%s', " % (matrix[strike].keys()[i], matrix[strike].keys()[i])
+        s = s + '],'
+        for month, strikes in sorted(matrix.iteritems()):
+            l = ''
+            for strike, cp in sorted(strikes.iteritems()):
+                l = l + '[%s,%s,%s,%s,%s],' % (strike, cp['P']['0'], cp['P']['1'], cp['P']['2'], cp['P']['3'])
+                
+                                
+            s = s + l + '],\n'
+        
+        print s
+        html_tmpl = html_tmpl.replace('{{{data}}}', s)
+        
+        return html_tmpl 
+ 
+    @cherrypy.expose
+    def opt_implv(self):
+        #r_host = cherrypy.request.app.config['redis']['redis.server']
+        #r_port = cherrypy.request.app.config['redis']['redis.port']
+        #r_db = cherrypy.request.app.config['redis']['redis.db']
+        #r_sleep = cherrypy.request.app.config['redis']['redis.sleep']
+        
+        opt_implv = cherrypy.request.app.config['redis']['redis.datastore.key.option_implv']
+        #rs = redis.Redis(r_host, r_port, r_db)
+        
+        rs = QServer.r_conn
+                
+        opt_implv_tmpl = '%s%s/opt-chains-tmpl.html' % (cherrypy.request.app.config['/']['tools.staticdir.root'], cherrypy.request.app.config['/static']['tools.staticdir.tmpl'])
+        f = open(opt_implv_tmpl)
+        html_tmpl = f.read()
+
+        s_dict = rs.get(opt_implv)
+        
+        # sample value
+        # {u'25400': {u'20150828': {u'P': [u'null', 1400.0], u'C': [0.21410911336791702, 29.0]}, u'20150929': {u'P': [u'null', u'null'], u'C': [0.1934532406732742, 170.0]}}, ...
+        matrix = json.loads(s_dict)
+        
+
+        
+        strike = matrix.keys()[0]
+        print matrix
+        num_months = len(matrix[strike])
+        s = '["strike",'
+        
+        sorted_months = sorted(matrix[strike].keys())
+        for i in range(num_months):          
+            s = s + "'P-%s', 'C-%s', " % (sorted_months[i], sorted_months[i])
+        s = s + '],'
+        for strike, items in sorted(matrix.iteritems()):
+            s = s + '[%s,' % str(strike)
+            l = ''
+            for month, cp in sorted(items.iteritems()):
+                print month, cp
+                l = l + ''.join('%s,%s,' % (cp['P'][0], cp['C'][0]))
+                                
+            s = s + l + '],\n'
+        
+        html_tmpl = html_tmpl.replace('{{{data}}}', s)
+
+
+
+        s = '["strike",'
+        for i in range(num_months):          
+            s = s + "'P-%s', 'C-%s', " % (sorted_months[i], sorted_months[i])
+        s = s + '],'
+        for strike, items in sorted(matrix.iteritems()):
+            s = s + '[%s,' % str(strike)
+            l = ''
+            for month, cp in sorted(items.iteritems()):
+                l = l + ''.join('%s,%s,' % (cp['P'][1], cp['C'][1]))
+                                
+            s = s + l + '],\n'
+        
+        
+        html_tmpl = html_tmpl.replace('{{{dataPremium}}}', s)
+
+        
+
+        
+        return html_tmpl
+
+
+
+
+    @cherrypy.expose
+    def opt_implv_ex(self):
+        #r_host = cherrypy.request.app.config['redis']['redis.server']
+        #r_port = cherrypy.request.app.config['redis']['redis.port']
+        #r_db = cherrypy.request.app.config['redis']['redis.db']
+        #r_sleep = cherrypy.request.app.config['redis']['redis.sleep']
+        
+        opt_implv = cherrypy.request.app.config['redis']['redis.datastore.key.option_implv']
+        #rs = redis.Redis(r_host, r_port, r_db)
+        
+        rs = QServer.r_conn
+                
+        opt_implv_tmpl = '%s%s/opt-chains-ex-tmpl.html' % (cherrypy.request.app.config['/']['tools.staticdir.root'], cherrypy.request.app.config['/static']['tools.staticdir.tmpl'])
+        f = open(opt_implv_tmpl)
+        html_tmpl = f.read()
+
+        s_dict = rs.get(opt_implv)
+        
+        # sample value
+        # {u'25400': {u'20150828': {u'P': [u'null', 1400.0], u'C': [0.21410911336791702, 29.0]}, u'20150929': {u'P': [u'null', u'null'], u'C': [0.1934532406732742, 170.0]}}, ...
+        matrix = json.loads(s_dict)
+        
+
+        
+        strike = matrix.keys()[0]
+        print matrix
+        num_months = len(matrix[strike])
+        s = '["strike",'
+        
+        sorted_months = sorted(matrix[strike].keys())
+        for i in range(num_months):          
+            s = s + "'P-%s', 'C-%s', " % (sorted_months[i], sorted_months[i])
+        s = s + '],'
+        for strike, items in sorted(matrix.iteritems()):
+            s = s + '[%s,' % str(strike)
+            l = ''
+            for month, cp in sorted(items.iteritems()):
+                print month, cp
+                l = l + ''.join('%s,%s,' % (cp['P'][0], cp['C'][0]))
+                                
+            s = s + l + '],\n'
+        
+        html_tmpl = html_tmpl.replace('{{{data}}}', s)
+
+
+
+        s = '[{label:"strikes",type:"number"},'
+        for i in range(num_months):          
+            s = s + "{label: 'P-%s', type:'number'},\
+                {label: 'Pb-%s', id:'i0', type:'number', role:'interval'},\
+                {label: 'Pa-%s', id:'i0', type:'number', role:'interval'},\
+                {label: 'C-%s', type:'number', },\
+                {label: 'Cb-%s', id:'i0', type:'number', role:'interval'},\
+                {label: 'Ca-%s', id:'i0',type:'number', role:'interval'},"\
+                 % (sorted_months[i], sorted_months[i],sorted_months[i], sorted_months[i],sorted_months[i], sorted_months[i])
+        s = s + '],'
+        for strike, items in sorted(matrix.iteritems()):
+            s = s + '[%s,' % str(strike)
+            l = ''
+            for month, cp in sorted(items.iteritems()):
+                l = l + ''.join('%s,%s,%s,%s,%s,%s,' % (cp['P'][1], cp['P'][2],cp['P'][3],cp['C'][1],cp['C'][2],cp['C'][3]))
+                                
+            s = s + l + '],\n'
+        
+        
+        html_tmpl = html_tmpl.replace('{{{dataPremium}}}', s)
+
+        
+
+        
+        return html_tmpl
+
+
+
+
+
+
+
+
+    @cherrypy.expose
+    def ws_cal_implvol(self, s, x, cp, ed, xd, r, d, v, p, out='iv'):
+        try:
+        #spot, strike, callput, evaldate, exdate, rate, div, vol, premium
+            rs = optcal.cal_implvol(float(s), float(x), cp, ed, xd, float(r), float(d), float(v), float(p))
+            return str(rs['imvol'])
+        except:
+            return
+
+    @cherrypy.expose
+    def ws_cal_option(self, s, x, cp, ed, xd, r, d, v, out='npv'):
+        #spot, strike, callput, evaldate, exdate, rate, div, vol
+        
+        keys = ['npv', 'delta', 'gamma', 'theta', 'vega'];
+        
+        try:
+            rs = optcal.cal_option(float(s), float(x), cp, ed, xd, float(r), float(d), float(v))
+            
+            if out == 'csv':
+                logging.debug('ws_cal_option: ' + ','.join(str(rs[s]) for s in keys))
+                return ','.join(str(rs[s]) for s in keys)
+            elif out == 'json':
+                return json.dumps(rs)
+            else:
+                return str(rs[out])  
+        except:
+            #exc_type, exc_value, exc_traceback = sys.exc_info()
+            return traceback.format_exc()
+            
+    @cherrypy.expose
+    def ws_get_hist_implv(self, dataAt):
+    # given a date string YYMMDDHHMM, this routine returns an array of
+    # implied vols arranged in a format like the below
+    #      [["strike",'P-20150828', 'C-20150828', 'P-20150929', 'C-20150929', ],
+    #                      [21800,0.29153118077,null,0.241032122988,null,],
+    #                      [22000,0.284002011642,null,0.238145680311,null,],
+    #                      [22200,0.270501965746,null,0.222647164832,null,]]   
+        pass
+    
+    
+    @cherrypy.expose
+    def ws_market_data(self, r_ckey, fid):
+        if str(fid).upper() == 'ALL':
+            return QServer.r_conn.get(r_ckey)
+        val = QServer.r_conn.get(r_ckey)
+        if val is None:
+            return 'invalid request. Check your input again!'
+        dict = json.loads(QServer.r_conn.get(r_ckey))
+        return str(dict[fid]) 
+ 
+    @cherrypy.expose
+    def ws_position_chart(self):
+        p = portfolio.PortfolioManager(config)
+        p.retrieve_position()
+        opt_pos_chart_tmpl = '%s%s/opt-pos-chart-tmpl.html' % (cherrypy.request.app.config['/']['tools.staticdir.root'], cherrypy.request.app.config['/static']['tools.staticdir.tmpl'])
+        f = open(opt_pos_chart_tmpl)
+        html_tmpl = f.read()
+        html_tmpl = html_tmpl.replace('{{{dataPCpos}}}', p.get_grouped_options_str_array())
+        
+        html_tmpl = html_tmpl.replace('{{{dataTablePos}}}', p.get_tbl_pos_csv())
+        
+        html_tmpl = html_tmpl.replace('{{{option_months}}}', ''.join(('%s, ' % m) for m in p.get_traded_months()))
+        v = p.group_pos_by_right()
+        html_tmpl = html_tmpl.replace('{{{PRvsCR}}}}', '%0.2f : %0.2f' % (v[0][1], v[1][1]))
+        
+        #print p.get_portfolio_summary()
+        #html_tmpl = html_tmpl.replace('{{{pos_summary}}}', ''.join('<li>%s:   %s</li>' % (x[0],x[1]) for x in p.get_portfolio_summary() ))
+        #print '\n'.join('%s:\t\t%s' % (k,v) for k,v in sorted(json.loads(DataMap.rs.get(port_key)).iteritems()))
+        
+        
+        return html_tmpl
+ 
+    @cherrypy.expose
+    def ws_position_summary(self):
+        p = portfolio.PortfolioManager(config)
+        keys = [("delta_1percent","number"),("delta_all","number"),("delta_c","number"),("delta_p","number"),\
+                ("theta_1percent","number"),("theta_all","number"),("theta_c","number"),("theta_p","number"),\
+                ("unreal_pl","number"),("last_updated","string"),("status","string")]
+        d = p.get_portfolio_summary()
+        
+        dict= {}
+        dict['cols'] = [{'label': x[0], 'type': x[1]} for x in keys]
+        dict['rows'] = [{'v': d[x[0]]} for x in keys]
+        print json.dumps(dict)
+        return json.dumps(dict)
+
+    @cherrypy.expose
+    def ws_recal_pos(self, force_refresh=False):
+        p = portfolio.PortfolioManager(config)
+        if force_refresh:
+            p.retrieve_position()
+        l_gmap = p.recal_port()
+        print l_gmap
+        return json.dumps(l_gmap)        
+        
+
+
+    @cherrypy.expose
+    def ws_pos_csv(self):
+        p = portfolio.PortfolioManager(config)
+        p.retrieve_position()
+        s = "%s" % p.get_tbl_pos_csv_old()
+        #s = "%s" % p.get_tbl_pos_csv()
+        #print s
+        s = s.replace(',[', '').replace(']', '<br>')
+        print s
+        return s[1:len(s)-3]
+    
+    @cherrypy.expose
+    def getSHquote(self, qs):
+
+#http://api.money.126.net/data/feed/0000001,1399001,1399300        
+#_ntes_quote_callback({"0000001":{"code": "0000001", "percent": 0.015468, "askvol1": 0, "askvol3": 0, "askvol2": 0, "askvol5": 0,
+# "askvol4": 0, "price": 2972.57, "open": 2978.03, "bid5": 0, "bid4": 0, "bid3": 0, "bid2": 0, "bid1": 0, "high": 3014.41, "low": 2929.0, 
+#"updown": 45.28, "type": "SH", "bidvol1": 0, "status": 0, "bidvol3": 0, "bidvol2": 0, "symbol": "000001", "update": "2015/08/27 12:43:00", 
+#"bidvol5": 0, "bidvol4": 0, "volume": 19800251400, "ask5": 0, "ask4": 0, "ask1": 0, "name": "\u4e0a\u8bc1\u6307\u6570", "ask3": 0, "ask2": 0,
+# "arrow": "\u2191", "time": "2015/08/27 12:42:57", "yestclose": 2927.29, "turnover": 204156106776} });        
+        url = 'http://api.money.126.net/data/feed/%s?callback=ne3587367b7387dc' % qs
+        print url
+        pg = urllib2.urlopen(url.encode('utf-8'))    
+        s = pg.read().replace('ne3587367b7387dc(', '')
+        
+        
+        s = s[:len(s)-2]
+        print s
+        return s      
+         
+if __name__ == '__main__':
+            
+#     logging.basicConfig(filename = "log/opt.log", filemode = 'a', 
+#                         level=logging.DEBUG,
+#                         format='%(asctime)s %(levelname)-8s %(message)s')      
+#  
+# 
+#     config = ConfigParser.ConfigParser()
+#     config.read("config/app.cfg")
+#     host = config.get("redis", "redis.server").strip('"').strip("'")
+#     port = config.get("redis", "redis.port")
+#     db = config.get("redis", "redis.db")    
+#     r_conn = redis.Redis(host,port,db)
+#     cherrypy.quickstart(QServer(r_conn, config), '/', "config/app.cfg")
+   
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]    
+    config = ConfigParser.ConfigParser()
+    if len(config.read(cfg_path)) == 0:      
+        raise ValueError, "Failed to open config file" 
+    
+    logconfig = eval(config.get("opt_serve", "opt_serve.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'
+    logging.basicConfig(**logconfig)            
+    host = config.get("redis", "redis.server").strip('"').strip("'")
+    port = config.get("redis", "redis.port")
+    db = config.get("redis", "redis.db")    
+    r_conn = redis.Redis(host,port,db)
+    
+    cherrypy.quickstart(QServer(r_conn, config), '/', cfg_path[0])
+    
+   

finopt/optcal.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+from QuantLib import *
+
+
+
+def cal_implvol(spot, strike, callput, evaldate, exdate, rate, div, vol, premium):
+    
+    Settings.instance().evaluationDate = str2qdate(evaldate)
+    exercise = EuropeanExercise(str2qdate(exdate))
+    payoff = PlainVanillaPayoff(str2qopt_type(callput), strike)
+    option = EuropeanOption(payoff,exercise)
+    S = QuoteHandle(SimpleQuote(spot))
+    r = YieldTermStructureHandle(FlatForward(0, TARGET(), rate, Actual360()))
+    q = YieldTermStructureHandle(FlatForward(0, TARGET(), div, Actual360()))
+    sigma = BlackVolTermStructureHandle(BlackConstantVol(0, TARGET(), vol, Actual360()))
+    process = BlackScholesMertonProcess(S,q,r,sigma)
+    im = 0.0
+    im = option.impliedVolatility(premium, process)
+    results = {}
+    results['imvol'] = im
+ 
+
+    return results
+
+
+def cal_option(spot, strike, callput, evaldate, exdate, rate, div, vol):
+    
+    Settings.instance().evaluationDate = str2qdate(evaldate)
+    exercise = EuropeanExercise(str2qdate(exdate))
+    payoff = PlainVanillaPayoff(str2qopt_type(callput), strike)
+    option = EuropeanOption(payoff,exercise)
+    S = QuoteHandle(SimpleQuote(spot))
+    r = YieldTermStructureHandle(FlatForward(0, TARGET(), rate, Actual360()))
+    q = YieldTermStructureHandle(FlatForward(0, TARGET(), div, Actual360()))
+    sigma = BlackVolTermStructureHandle(BlackConstantVol(0, TARGET(), vol, Actual360()))
+    process = BlackScholesMertonProcess(S,q,r,sigma)
+    engine = AnalyticEuropeanEngine(process)
+    option.setPricingEngine(engine)
+            
+    results = {}
+    results['npv'] = option.NPV()
+
+    results['delta'] = option.delta()
+    results['gamma'] = option.gamma()
+    
+    results['theta'] = option.theta() / 360
+    results['vega'] = option.vega()    
+
+ 
+
+    return results
+    
+        
+
+
+def str2qdate(yyyymmdd):
+    months = [January, February, March, April, May, June, July, August, September, October,
+                November, December]
+    #print '%d%d%d'% (int(yyyymmdd[6:8]), int(yyyymmdd[4:6])-1 , int(yyyymmdd[0:4])) 
+    return Date(int(yyyymmdd[6:8]), months[int(yyyymmdd[4:6])-1 ], int(yyyymmdd[0:4]))
+    
+
+def str2qopt_type(callput):
+    if callput.upper() == 'C':
+        return Option.Call
+    return Option.Put
+
+if __name__ == '__main__':
+    # todaysDate = Date(10,August, 2015)
+    # Settings.instance().evaluationDate = todaysDate
+    # 
+    # exercise = EuropeanExercise(Date(28,August,2015))
+    # payoff = PlainVanillaPayoff(Option.Call, 25600.0)
+    # option = EuropeanOption(payoff,exercise)
+    # 
+    # v = 0.19
+    # S = QuoteHandle(SimpleQuote(24507.0))
+    # r = YieldTermStructureHandle(FlatForward(0, TARGET(), 0.0005, Actual360()))
+    # q = YieldTermStructureHandle(FlatForward(0, TARGET(), 0.0005, Actual360()))
+    # sigma = BlackVolTermStructureHandle(BlackConstantVol(0, TARGET(), v, Actual360()))
+    # process = BlackScholesMertonProcess(S,q,r,sigma)
+    # 
+    # 
+    # im = option.impliedVolatility(85.0, process)
+    # print im
+    # 
+    # engine = AnalyticEuropeanEngine(process)
+    # option.setPricingEngine(engine)
+    # print '%0.4f delta=%0.4f gamma=%0.4f theta=%0.4f vega=%0.4f ' % (option.NPV(), option.delta(), option.gamma(), option.theta() / 360, option.vega())
+    
+    
+    # for i in range(24000, 27000, 200):
+    #     results = cal_option(24189.0, i * 1.0, 'C', '20150804', '20150828', 0.0005, 0.0005, 0.19)
+    #     results2 = cal_option(24189.0, i * 1.0, 'P', '20150804', '20150828', 0.0005, 0.0005, 0.19)
+    #     print ('%d: '% i) + ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems()) + '|' +  ''.join ('%s=%0.4f, '%(k,v) for k, v in results2.iteritems())
+    # for k, v in results.iteritems():
+    #     print '%s= %0.4f' % (k, v)
+    
+    
+    #spot 24119.0, X 25000, right: P, evaldate: 20150812, expiry: 20150828, rate: 0.0005, div: 0.0005, vol: 0.2000, premium: 334.0000
+    #spot 24149.0, X 25200, right: P, evaldate: 20150812, expiry: 20150828, rate: 0.0005, div: 0.0005, vol: 0.2000, premium: 437.5000
+    
+    results = cal_option(24149.0, 25200, 'P', '20150812', '20150828', 0.0005, 0.0005, 0.19)
+    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+    results = cal_implvol(24149.0, 25200, 'P', '20150812', '20150828', 0.0005, 0.0005, 0.19, 1247)
+    
+        
+    print ''.join ('%s=%0.4f, '%(k,v) for k, v in results.iteritems())
+

finopt/options_data.py

@@ -0,0 +1,975 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import thread
+import ConfigParser
+from ib.ext.Contract import Contract
+from ib.opt import ibConnection, message
+from time import sleep
+import time, datetime
+import optcal
+import opt_serve
+import cherrypy
+import redis
+import ystockquote
+import portfolio
+            
+# Tick Value      Description
+# 5001            impl vol
+# 5002            delta
+# 5003            gamma
+# 5004            theta
+# 5005            vega
+# 5006            premium
+# IB tick types code reference
+# https://www.interactivebrokers.com/en/software/api/api.htm
+class OptionsMarketDataManager():
+    
+    IMPL_VOL = 5001
+    DELTA    = 5002
+    GAMMA    = 5003
+    THETA    = 5004
+    VEGA     = 5005
+    PREMIUM  = 5006
+    
+
+    
+    # 
+    # map to futures ticker ids key:contract_mth  val: tick_id undlys ={'20150828': 1} 
+    undlys = {}
+    
+
+    # market data manager reference
+    mdm = None
+    rs = None
+    config = {}
+    
+    cal_greeks_config = {}
+
+    def __init__(self, config, mdm):
+        
+        self.mdm = mdm
+        self.mdm.setOMDM(self)
+        self.config = config
+
+        # config file sample values        
+        # option.underlying = "('HSI', 'FUT', 'HKFE', 'HKD', '', 0, '')"
+        # option.underlying.month_price = "[['20150828', 22817.0], ['20150929', 22715.0]]"
+        # option.underlying.tick_size = 200
+        # option.chain_range = 0.08
+        
+        undly_tuple = eval(config.get("market", "option.underlying").strip('"').strip("'"))
+        undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+        undly_yahoo_ws = eval(config.get("market", "option.underlying.yahoo_ws").strip('"').strip("'"))
+        self.cal_greeks_config = eval(config.get("market", "option.greeks.recal").strip('"').strip("'"))
+    
+        
+        
+        undly_tick_size = int(config.get("market", "option.underlying.tick_size"))
+        undly_chain_range = float(config.get("market", "option.chain_range"))
+    
+        logging.debug("OptionsMarketDataManager --------------")
+        logging.debug("OptionsMarketDataManager init: undly_tuple %s" % str(undly_tuple))
+        logging.debug("OptionsMarketDataManager init: undly_months_prices %s" % str(undly_months_prices))
+        logging.debug("OptionsMarketDataManager init: undly_tick_size %d" % undly_tick_size)
+        logging.debug("OptionsMarketDataManager init: undly_chain_range %f" % undly_chain_range)
+        logging.debug("OptionsMarketDataManager --------------")
+        
+        self.option_range(undly_tuple, undly_months_prices, undly_yahoo_ws, undly_tick_size, undly_chain_range)
+
+
+    # 20150820 fix previous logic which onlyu support near month underlying subscription 
+    #def option_range(self, contractTuple, months, undlypx, tsize, bound):
+    def option_range(self, contractTuple, months_prices, undly_yahoo_ws, tsize, bound):
+        
+
+        for mth_price in months_prices:
+            
+            # set up the underlying structure
+            # fix the contract month value
+            undly = []
+            for e in contractTuple:
+                undly.append(e)
+            # set month value
+            undly[4] = mth_price[0] 
+
+
+            # subscribe the underlying
+            undly_tick_id = self.mdm.subscribe_tuple(tuple(undly))
+            # store the tick id for lookup later
+            self.undlys[mth_price[0]] = undly_tick_id
+             
+            logging.debug('OptionsMarketDataManager: option_range >>>>> subscribe underlying contract: %s' % undly)
+            
+            
+            
+            px = float(eval('%s("%s")' % (undly_yahoo_ws['func'], mth_price[2]))) if (undly_yahoo_ws['use_yahoo'] == True) else mth_price[1]
+            logging.info("OptionsMarketDataManager: ********************")
+            logging.warn("OptionsMarketDataManager: *** ")
+            logging.info("OptionsMarketDataManager: ***")
+            logging.info("OptionsMarketDataManager: *** Initial underlying price used: %.2f " % px)
+            logging.info("OptionsMarketDataManager: *** Price obtained from %s->%s" % ("yahoo" if undly_yahoo_ws['use_yahoo'] else "config file", \
+                                                                                       undly_yahoo_ws['func']))
+            logging.info("OptionsMarketDataManager: ***")
+            logging.info("OptionsMarketDataManager: ***")
+            logging.info("OptionsMarketDataManager: ********************")            
+            undlypx = round(px  / tsize) * tsize
+            upper_limit = undlypx * (1 + bound)
+            lower_limit = undlypx * (1 - bound)        
+            
+            
+            logging.info("OptionsMarketDataManager: *** subscription boundaries between %0.4f and %0.4f" % (lower_limit, upper_limit))
+            
+            # send to IB conn to subscribe
+            for i in range(int(undlypx), int(upper_limit ), tsize):
+                self.add_subscription(contractTuple, undly[4], i, 'C')
+                self.add_subscription(contractTuple, undly[4], i, 'P')
+             
+            for i in range(int(undlypx) - tsize, int(lower_limit), -tsize):
+                self.add_subscription(contractTuple, undly[4], i, 'C')
+                self.add_subscription(contractTuple, undly[4], i, 'P')            
+
+
+    def add_subscription(self, contractTuple, month, X, right):
+        
+        #contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20150828', 0, '')
+        opt = []
+        for e in contractTuple:
+            opt.append(e)
+        opt[1] = 'OPT'
+        opt[4] = month
+        opt[5] = X if isinstance(X, int) else int(X)
+        opt[6] = right
+        id = self.mdm.subscribe_tuple(tuple(opt))
+        logging.debug('OptionsMarketDataManager: add_subscription: %s, tick_id %d' % (opt, id))
+        return id
+        
+    
+    def determine_premium(self, msg):    
+
+
+        premium = None
+        
+        
+        #if msg.field == 4:
+        if msg.field in [4,9]:
+            premium = msg.price
+            return premium
+            
+        if msg.field in [1,2]: #bid
+            
+            
+                    
+            if msg.price < 0:
+                logging.debug("OptionsMarketDataManager determine_premium: received bogus price from IB feed. %0.4f" % msg.price)
+                logging.debug("OptionsMarketDataManager determine_premium: attempt to use last price in datamap.")
+                if 4 in DataMap().get(msg.tickerId):
+                    last_price = DataMap().get(msg.tickerId)[4]
+                    if last_price > 0:
+                        premium = last_price
+                        logging.debug("OptionsMarketDataManager determine_premium: using last price %0.4f" % premium)
+                        return premium
+                logging.debug("OptionsMarketDataManager determine_premium: No last price, no premium derived.")
+                return None
+        else:
+            logging.debug("OptionsMarketDataManager determine_premium: price field not in 1,2 or 4. skip this message...")
+            return None
+        
+        #   
+        # at this point, msg price is > 0 and msg.field in 1 or 2
+        #
+        
+        def derive_mid_price(field_id):
+            bidask = 2 if (field_id == 1) else 1
+            if bidask in DataMap().get(msg.tickerId):
+                if DataMap().get(msg.tickerId)[bidask] > 0.0:
+                    mid = (DataMap().get(msg.tickerId)[bidask] + msg.price) / 2
+                else:
+                    mid = msg.price
+            logging.debug("OptionsMarketDataManager: incoming price field%d == %0.4f datamap field%d ==2 %0.4f"\
+                           % (field_id, msg.price, bidask, DataMap().get(msg.tickerId)[bidask]))                                 
+            return mid
+        
+        premium = derive_mid_price(msg.field)
+        
+                    
+        if premium is None:
+            # skip calculation because we don't have a valid value for premium 
+            logging.debug("OptionsMarketDataManager: derive_mid_price: unable to derive a usable premium for ticker id %s, skipping computation" % (msg.tickerId))
+            return None
+        
+
+                
+            
+            
+    def on_tick_data_changed(self, msg):
+        #print 'omdm %s' % msg
+        if msg.typeName in ['tickPrice', 'tickSize']:
+           
+            if msg.typeName == 'tickPrice':
+                
+#                 if self.isTickIdAnOption(msg.tickerId):
+#                 
+#                     contract = DataMap().get(msg.tickerId)['contract']
+#                     
+#                     
+#                     
+#                     logging.info("OptionsMarketDataManager: on_tick_data_changed: received msg field %s for ticker id %s" % (msg.field, msg.tickerId))
+#                     # last price%Y%m%d
+#                     premium = None
+#                     if msg.price < 0:
+#                         logging.debug("OptionsMarketDataManager: received bogus price from IB feed. %0.4f" % msg.price) 
+#                         return
+#                     
+#                     if msg.field == 4:
+#                         premium = msg.price    
+#                     elif msg.field == 1: #bid
+#                         if 2 in DataMap().get(msg.tickerId):
+#                             if DataMap().get(msg.tickerId)[2] > 0.0:
+#                                 premium = (DataMap().get(msg.tickerId)[2] + msg.price) / 2
+#                             else:
+#                                 premium = msg.price
+#                                 
+#                             logging.debug("OptionsMarketDataManager: msgfiled ==1 %0.4f msgfiled ==2 %0.4f" % (msg.price, DataMap().get(msg.tickerId)[2]))                                 
+#                     elif msg.field == 2: #ask
+#                         if 1 in DataMap().get(msg.tickerId):
+#                             if DataMap().get(msg.tickerId)[1] > 0.0:
+#                                 premium = (DataMap().get(msg.tickerId)[1] + msg.price) / 2
+#                             else:
+#                                 premium = msg.price
+#                             logging.debug("OptionsMarketDataManager: msgfiled ==2 %0.4f msgfiled ==1 %0.4f" % (msg.price, DataMap().get(msg.tickerId)[1]))                                
+#                     if premium is None:
+#                         # skip calculation because we don't have a valid value for premium 
+#                         logging.debug("OptionsMarketDataManager: on_tick_data_changed: unable to derive a usable premium for ticker id %s, skipping computation" % (msg.tickerId))
+#                         return
+                if self.isTickIdAnOption(msg.tickerId):
+                
+                    contract = DataMap().get(msg.tickerId)['contract']
+                    logging.info("OptionsMarketDataManager: on_tick_data_changed: received msg field %s for ticker id %s" % (msg.field, msg.tickerId))                    
+                            
+                    premium = self.determine_premium(msg)
+                    if premium == None:
+                        return
+                    
+                    
+                    
+                    undly = DataMap().get(self.getUndlyId(contract.m_expiry))
+                    if 4 in undly:  # the last price
+                        spot = undly[4]
+                        
+                        logging.info('OptionsMarketDataManager:on_tick_data_changed: undelying spot %0.4f of month %s' % (spot, contract.m_expiry))
+                        today = time.strftime('%Y%m%d') 
+                        logging.info('OptionsMarketDataManager:on_tick_data_changed: today %s ' % time.strftime('%Y%m%d'))
+                        div = self.cal_greeks_config['div']
+                        rate = self.cal_greeks_config['rate']
+                        
+                        # vol is not used in the calculation of implv but quantlib requires the parameter to be passed
+                        vol = self.cal_greeks_config['vol']
+                        logging.info('OptionsMarketDataManager:on_tick_data_changed: symbol %s, spot %s, X %s, right: %s, evaldate: %s, expiry: %s, rate: %0.4f, div: %0.4f, vol: %0.4f, premium: %0.4f' % 
+                                        (contract.m_symbol, spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate, div, vol, premium))
+                        
+                        try:
+                        
+                            iv = optcal.cal_implvol(spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate, div, vol, premium)
+                            
+                        except Exception, err:
+                            logging.error(traceback.format_exc())
+                            logging.error("OptionsMarketDataManager: *******************recovering from a implvol error ******")
+                            
+                            intrinsic = abs(contract.m_strike - spot)  
+                            iv = optcal.cal_implvol(spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate, div, vol, premium)
+                            logging.error("OptionsMarketDataManager: ******** Using intrinsic value to calculate premium %0.4f instead of the spot premium %0.4f"\
+                                                                        % (intrinsic, premium ))
+                        
+                        logging.info('OptionsMarketDataManager:on_tick_data_changed: implied vol: %0.4f' % iv['imvol'])
+                        results = optcal.cal_option(spot, contract.m_strike, contract.m_right, today, contract.m_expiry, rate, div, iv['imvol'])
+                        
+                        DataMap().get(msg.tickerId)[OptionsMarketDataManager.IMPL_VOL] = iv['imvol']
+                        DataMap().get(msg.tickerId)[OptionsMarketDataManager.DELTA] = results['delta']
+                        DataMap().get(msg.tickerId)[OptionsMarketDataManager.GAMMA] = results['gamma']
+                        DataMap().get(msg.tickerId)[OptionsMarketDataManager.THETA] = results['theta']
+                        DataMap().get(msg.tickerId)[OptionsMarketDataManager.VEGA] = results['vega']
+                        DataMap().get(msg.tickerId)[OptionsMarketDataManager.PREMIUM] = results['npv'] 
+        
+                        # update Redis store
+                        #
+                        DataMap().update_rd(msg.tickerId)
+        
+                else: # underlying price changed
+                    # check whether new option chains need to be added
+                    #
+                    # check logic to be implemented
+                    
+                    
+                    # save undly tick to REDIS
+                    contract = DataMap().get(msg.tickerId)['contract']
+                    logging.debug('OptionsMarketDataManager:on_tick_data_changed: ------- Underlying Price updated: tick id %d datamap contract %s' % (msg.tickerId, ContractHelper.printContract(contract)))
+                    
+                    
+                    DataMap().update_rd(msg.tickerId)
+          
+    
+    # return tick_id of underlying 
+    # this function has problem, it doesn't cater for the symbol type
+    # so MHI contracts will be processed just fine but with HSI futures  
+    # returned as its underlying
+    # >>> ok as it doesn't casuse any problem in calculating the greeks
+    # >>> but need to be fixed later
+    def getUndlyId(self, contract_mth):
+        return self.undlys[contract_mth]
+    
+    def isTickIdAnOption(self, tickid):
+        return tickid not in self.undlys.values()
+            
+
+class MarketDataManager():
+    
+    config = {}
+    tick_id = 0
+    instrus = {}
+    con = None
+    omdm = None
+    
+    def __init__(self, config):
+        self.config = config
+        ibgw = config.get("market", "ib.gateway").strip('"').strip("'")
+        ibport = config.get("market", "ib.port")
+        ibid = config.get("market", "ib.appid")
+        logging.debug("ibgw, port, app id: %s %s %s" % (ibgw, ibport, ibid))
+        self.con = ibConnection(ibgw, int(ibport), int(ibid))
+        
+        self.con.registerAll(self.on_ib_message)
+
+    
+
+    def setOMDM(self, omdm):
+        self.omdm = omdm
+        self.omdm.mdm = self
+        
+    def subscribe_tuple(self, tuple):        
+        return self.subscribe(ContractHelper.makeContract(tuple))
+
+    def subscribe(self, contract):
+
+
+        tick_id = -1
+        if self.isContractSubscribed(contract) == False:
+            #logging.debug( 'MarketDataManager:subscribe subscrbing to' + ContractHelper.printContract(contract))
+            
+            self.con.reqMktData(self.tick_id, contract, '', False)
+            tick_id = self.tick_id
+            #self.instrus[self.tick_id] = {}
+            #self.instrus[self.tick_id]['contract'] = contract
+            
+            DataMap().set(self.tick_id, {})
+            DataMap().get(self.tick_id)['contract'] = contract
+            
+            logging.debug( 'MarketDataManager:subscribe DataMap stored value - >' + ContractHelper.printContract(DataMap().get(self.tick_id)['contract']))
+            self.tick_id = self.tick_id + 1
+            
+        else:
+            logging.debug("Contract has been subscribed already %s" % ContractHelper.printContract(contract))
+        return tick_id
+
+    def isContractSubscribed(self, contract):
+        
+        
+        
+        #for c in self.instrus.values():
+        # 20150814 - changed to rely on DataMap values
+        for c in DataMap().getData().values():
+            if c['contract'] == contract:
+                logging.debug("MarketDataManager: isContractSubscribed: YES %s" % ContractHelper.printContract(c['contract']))
+                return True   
+        
+        logging.debug("MarketDataManager: isContractSubscribed: No. Subscribing to %s" % ContractHelper.printContract(contract))
+        return False
+
+
+        
+
+    def on_ib_message(self, msg):
+        
+        if msg.typeName in ['tickPrice', 'tickSize']:
+
+#             if msg.tickerId not in self.instrus:
+#                 self.instrus[msg.tickerId] = {}
+            
+            if msg.typeName == 'tickPrice':
+                #self.instrus[msg.tickerId][msg.field] = msg.price
+                DataMap().get(msg.tickerId)[msg.field] = msg.price
+                 
+            if msg.typeName == 'tickSize':
+                #self.instrus[msg.tickerId][msg.field] = msg.size
+                DataMap().get(msg.tickerId)[msg.field] = msg.size
+            
+            #if msg.typeName == "tickSize":
+            #    logging.debug(msg)
+            
+            
+        if self.omdm != None:
+            self.omdm.on_tick_data_changed(msg)
+
+
+                               
+
+    def connect(self):
+        rc = self.con.connect()
+        logging.info("-----------------")
+        logging.info("-----------MarketDataManager ----Connect to IB connection: status: %s" % str(rc))
+        logging.info("-----------------")
+        if rc == False:
+            logging.error("-----------MarketDataManager ---- Connect to IB Failed!!! Terminating....")
+            sys.exit(-1)
+        
+        
+    def disconnect(self):
+        for i in range(self.tick_id):
+            self.con.cancelMktData(i)
+            logging.debug("cancelling tick id subscription %d" % i)
+        self.con.disconnect()
+
+class ContractHelper():
+    
+    def __init__(self, contractTuple):
+        self.makeContract(contractTuple)
+    
+    @staticmethod
+    def makeContract(contractTuple):
+        newContract = Contract()
+        newContract.m_symbol = contractTuple[0]
+        newContract.m_secType = contractTuple[1]
+        newContract.m_exchange = contractTuple[2]
+        newContract.m_currency = contractTuple[3]
+        newContract.m_expiry = contractTuple[4]
+        newContract.m_strike = contractTuple[5]
+        newContract.m_right = contractTuple[6]
+        logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s:' % contractTuple)
+        return newContract
+    
+    @staticmethod
+    def convert2Tuple(newContract):
+        newContractTuple = (newContract.m_symbol,\
+                            newContract.m_secType,\
+                            newContract.m_exchange,\
+                            newContract.m_currency,\
+                            newContract.m_expiry,\
+                            newContract.m_strike,\
+                            newContract.m_right, newContract.m_conId)
+        logging.debug( 'Contract Values:%s,%s,%s,%s,%s,%s,%s %s:' % newContractTuple)
+        return newContractTuple
+    
+    
+    @staticmethod
+    def printContract(contract):
+        s = '[%s-%s-%s-%s-%s-%s-%s-%s]' % (contract.m_symbol,
+                                                       contract.m_secType,
+                                                       contract.m_exchange,
+                                                       contract.m_currency,
+                                                       contract.m_expiry,
+                                                       contract.m_strike,
+                                                       contract.m_right,
+                                                       contract.m_conId)
+        #logging.info(s)
+        return s
+    
+    @staticmethod
+    def makeRedisKey(contract):
+        #print "makerediskey %s" % ContractHelper.printContract(contract)
+#20150904        
+        contract.m_strike = int(contract.m_strike)
+        
+        if contract.m_secType == 'OPT':
+            s = '%s-%s-%s-%s' % (contract.m_symbol,
+                                                           contract.m_expiry,
+                                                           contract.m_strike,
+                                                           contract.m_right)
+        else:
+            s = '%s-%s-%s-%s' % (contract.m_symbol,
+                                                           contract.m_expiry,
+                                                           contract.m_secType, '')
+            
+        return s
+      
+    @staticmethod
+    def makeRedisKeyEx(contract, old=False):
+        # this routine is to circumvent a problem in makeRedisKey for which 
+        # the key's position 3 have different meanings
+        # according to a particular m_secType, which is not right. 
+        # 
+        # to differentiate the keys generated by the old and new functions, 
+        # contract keys created using this routine have their last slot
+        # hard coded a magic number 102
+        
+        if (old):
+            return ContractHelper.makeRedisKey(contract)
+        
+        contract.m_strike = int(contract.m_strike)
+        s = '%s-%s-%s-%s-%s-%s-%d' % (contract.m_symbol,
+                                                           contract.m_expiry,
+                                                           contract.m_strike,
+                                                           contract.m_right,
+                                                           contract.m_secType,
+                                                           contract.m_currency,
+                                                           102)
+        return s
+      
+def test():
+#     contractTuple = ('USD', 'CASH', 'IDEALPRO', 'JPY', '', 0.0, '')
+#     contractTuple2 = ('xSD', 'CASH', 'IDEALPRO', 'JPY', '', 0.0, '')
+#      
+#     print 1 if contractTuple == contractTuple2 else -1
+#     
+#     
+#     
+#     contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '20150929', 0, '')
+#     
+#     
+#     d = DataMap()
+#     d.set('c', ContractHelper.makeContract(contractTuple))
+#     c = d.get('c')
+#     print ContractHelper.printContract(c)
+#     mdm = MarketDataManager({})
+#     
+#     o = OptionsMarketDataManager(mdm, c)
+    
+    def f1(*a):
+        s = ''
+        for i in a:
+            s = s + '%s,' % i
+        return s
+
+    rs = redis.Redis()
+    start = 201508200930
+    end = int(datetime.datetime.now().strftime('%Y%m%d%H%M'))
+    
+    for i in range(start, end):
+        if rs.exists(i):
+            j= json.loads(rs.get(i))
+            
+            i = str(i)
+# 
+#             print '[new Date(%s,%s,%s,%s,%s), %s, %s, %s, undefined],' % (i[0:4], i[4:6], i[6:8], i[8:10], i[10:12],
+#                                                         j['22400']['20150828']['P'][0], 
+#                                                         j['22600']['20150828']['P'][0],
+#                                                          j['22600']['20150929']['P'][0])
+            s =  '[new Date(%s,%d,%s,%s,%s),' % (i[0:4], int(i[4:6])-1, i[6:8], i[8:10], i[10:12])
+            s = s + f1(j['21400']['20150828']['P'][0], j['21800']['20150828']['P'][0], j['21800']['20150929']['P'][0], 'undefined', ']')
+            print s
+
+    
+
+    sys.exit(-1)
+
+class DataMap():
+    instrus = {}
+    rs = None
+    rskeys = {}
+    mkt_sessions = {}
+    config = None
+
+
+    def init_redis(self, config):
+        #self.config = config
+        DataMap.config = config
+        host = config.get("redis", "redis.server").strip('"').strip("'")
+        port = config.get("redis", "redis.port")
+        db = config.get("redis", "redis.db")
+        self.rskeys['redis.datastore.key.option_implv'] = config.get("redis", "redis.datastore.key.option_implv").strip('"').strip("'")
+        self.rskeys['redis.datastore.key.option_chains'] = config.get("redis", "redis.datastore.key.option_chains").strip('"').strip("'")
+        self.rskeys['redis.datastore.key.option_set'] = config.get("redis", "redis.datastore.key.option_set").strip('"').strip("'")
+        self.rskeys['redis.datastore.key.option_implv_ts_set'] = config.get("redis", "redis.datastore.key.option_implv_ts_set").strip('"').strip("'")
+        self.rskeys['redis.datastore.key.option_implv_ts'] = config.get("redis", "redis.datastore.key.option_implv_ts").strip('"').strip("'")
+        
+        
+        self.rskeys['hkex.openhours'] = config.get("market", "hkex.openhours").strip('"').strip("'")
+        self.rskeys['option.bid_ask_spread_tolerance'] = config.get("market", "option.bid_ask_spread_tolerance")
+        
+        DataMap.mkt_sessions = json.loads(self.rskeys['hkex.openhours'])
+        
+
+        
+        DataMap.rs = redis.Redis(host, port, db)
+        logging.info(self.rs.info())
+    
+    def redisConn(self):
+        return DataMap.rs
+    
+    
+    # this routine clones an element in instrus but without the contract object  
+    def simple_clone(self, contract):
+        instru = {}
+        for key, stuff in contract.iteritems():
+            if key <> 'contract':
+                instru[key] = stuff
+
+        return instru
+        
+        
+    def update_rd(self, tickerId):
+        
+        # this routine saves options data into redis map
+        # map structure
+        # 
+        #key:
+        #    underlying
+        #         month
+        #            strike
+        #                right
+        #value:
+        #                values
+        rs = DataMap().redisConn()
+        if rs == None:
+            logging.error("update_rd: Redis connection is None! Has it been initialized?")
+            return
+    
+        contract = DataMap().get(tickerId)['contract']
+        r_key = ContractHelper.makeRedisKey(contract)
+        
+        rs.sadd(self.rskeys['redis.datastore.key.option_set'], r_key)
+        data = DataMap().simple_clone(DataMap().get(tickerId))
+        data['last_updated'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+        
+        jd = json.dumps(data)
+        rs.set(r_key, jd)
+
+        #logging.debug( "update rd: [%s] %s " % (r_key, jd)) 
+        
+    
+    
+    def get(self, key):
+        return DataMap.instrus[key]
+    
+    def set(self, key, val):
+        DataMap.instrus[key] = val
+        
+    
+    def getData(self):
+        return DataMap.instrus
+    
+    
+    def isKeyInMap(self, k):
+        return (k in DataMap.instrus)
+    
+    
+    def createOptionsPanelData(self):
+
+        # this function dumps the data in DataMap and turn it into a json string
+        # set the value on redis map
+        # and make it available for the web service to query the json string
+        # it is up to the receiving end to format the json sring into whatever 
+        # structure for viewing. for example, into a data array accepted by google charts
+      
+        matrix = {}
+        for tick_id, instr_v in DataMap().getData().iteritems():
+#20150910 - data now contains MHI 
+# need to add extra condition to filter out MHI
+
+            #if DataMap().get(tick_id)['contract'].m_secType == 'OPT':
+            if DataMap().get(tick_id)['contract'].m_secType == 'OPT' and \
+                DataMap().get(tick_id)['contract'].m_symbol == 'HSI':
+                strike = DataMap().get(tick_id)['contract'].m_strike
+                expiry = DataMap().get(tick_id)['contract'].m_expiry
+                right = DataMap().get(tick_id)['contract'].m_right
+                if expiry not in matrix:
+                    matrix[expiry] = {}
+                if strike not in matrix[expiry]:
+                    matrix[expiry][strike] = {}
+                if right not in matrix[expiry][strike]:
+                    matrix[expiry][strike][right] = {}
+
+                for field, value in sorted(instr_v.iteritems()):
+                    if type(value) is Contract:
+                        pass
+                        #s = s + "%s," % ContractHelper.printContract(value)
+                    else:
+                        matrix[expiry][strike][right][field] = value
+                     
+                    
+        
+        
+        DataMap.rs.set(DataMap.rskeys['redis.datastore.key.option_chains'], json.dumps(matrix))
+        logging.debug("createOptionsPanelData: %s" % str( matrix))
+    
+    def printMap(self):
+
+        for tick_id, instr_v in DataMap().getData().iteritems():
+            s = 'Tick:%s [%s]>>' % (tick_id, ContractHelper.printContract(DataMap().get(tick_id)['contract'])) 
+            for field, value in sorted(instr_v.iteritems()):
+                if type(value) is Contract:
+                    continue
+                    #s = s + "%s," % ContractHelper.printContract(value)
+                else:
+                    s = s + "%d:[%s], " % (field, value)
+                 
+            print "%s" % s
+
+    def printMapCSV(self):
+        keys = [0,1,2,3,4,5,6,7,8,9,14,5001,5002,5003,5004,5005,5006]
+        print 'tick id,contract,' + ','.join(str(k) for k in keys)
+        for tick_id, instr_v in DataMap().getData().iteritems():
+            
+            #s = '%s,%s-%s-%s,' % (tick_id, DataMap().get(tick_id)['contract'].m_right, DataMap().get(tick_id)['contract'].m_expiry, DataMap().get(tick_id)['contract'].m_strike)
+            s = '%s,%s,' % (tick_id, ContractHelper.makeRedisKey(DataMap().get(tick_id)['contract']))
+            for k in keys:
+                if k in instr_v.keys():
+                    s = s + "%s, " % (instr_v[k])
+                else:
+                    s = s + ','
+                 
+            print "%s" % s
+        
+        
+    def createImplVolChartData(self):    
+
+        
+        matrix = {}
+        for tick_id, instr_v in DataMap().getData().iteritems():
+#            if DataMap().get(tick_id)['contract'].m_secType == 'OPT':
+            if DataMap().get(tick_id)['contract'].m_secType == 'OPT' and \
+                DataMap().get(tick_id)['contract'].m_symbol == 'HSI':
+                
+                strike = DataMap().get(tick_id)['contract'].m_strike
+                expiry = DataMap().get(tick_id)['contract'].m_expiry
+                right = DataMap().get(tick_id)['contract'].m_right
+                if strike not in matrix:
+                    matrix[strike] = {}
+                if expiry not in matrix[strike]:
+                    matrix[strike][expiry] = {}
+                if right not in matrix[strike][expiry]:
+#                    matrix[strike][expiry][right] = ['null','null']
+#20150915                     
+                    matrix[strike][expiry][right] = ['null','null','null','null']
+                if 5001 in instr_v.keys(): 
+                    matrix[strike][expiry][right][0] =  (instr_v[5001]) if self.isfloat(instr_v[5001]) else 'null'
+                # if bid and ask is available show mid price, else use last price
+                # but first verify that bid ask spread are not wider than 10%, else use last price
+                
+                
+                if 1 in instr_v.keys() and 2 in instr_v.keys() and 4 in instr_v.keys():
+                    
+                    bid = float(instr_v[1])
+                    ask = float(instr_v[2])
+                    mid = (bid + ask) / 2.0
+                    tolerance = float(self.rskeys['option.bid_ask_spread_tolerance'])
+                    logging.debug( "createImplVolChartData %s %s %s %s %s %s bid/ask ratio %f tolerance level %f" %\
+                                         (strike, expiry, right, instr_v[1], instr_v[2], instr_v[4], mid / max(bid, ask),\
+                                          tolerance)) 
+                    if  (mid / max(bid, ask)) >  tolerance and bid > 0 and ask > 0:
+                        matrix[strike][expiry][right][1] = str(mid)
+                        
+                        
+                    else:
+                        matrix[strike][expiry][right][1] =  (instr_v[4]) if self.isfloat(instr_v[4]) else 'null'
+                        logging.debug( 'createImplVolChartData: unusable bid/ask prices. using last price instead %s' % (instr_v[4]))
+                        
+#20150915     
+# add bid / ask to the the matrix
+# [0] delta, [1] mid or last, [2]                                    
+                    matrix[strike][expiry][right][2] =  (instr_v[1] if instr_v[1] > 0 else matrix[strike][expiry][right][1]) if self.isfloat(instr_v[1]) else 'null'
+                    matrix[strike][expiry][right][3] =  (instr_v[2] if instr_v[2] > 0 else matrix[strike][expiry][right][1]) if self.isfloat(instr_v[2]) else 'null'
+                    logging.debug( 'createImplVolChartData: unusable bid/ask prices. bid and ask %s,%s' % \
+                                   (str(matrix[strike][expiry][right][2]), str(matrix[strike][expiry][right][2])))
+                        
+                else:
+                    if 4 in instr_v.keys():
+                        #print "%s %s %s " % (instr_v[1], instr_v[2], instr_v[4]) 
+                        matrix[strike][expiry][right][1] =  (instr_v[4]) if self.isfloat(instr_v[4]) else 'null'
+                        matrix[strike][expiry][right][2] =  (instr_v[4]) if self.isfloat(instr_v[4]) else 'null'
+                        matrix[strike][expiry][right][3] =  (instr_v[4]) if self.isfloat(instr_v[4]) else 'null'
+        
+        DataMap.rs.set(DataMap.rskeys['redis.datastore.key.option_implv'], json.dumps(matrix))
+        logging.debug("createImplVolChartData: %s" % str( matrix))
+        
+        # save impl vol to historical time series
+        tk = datetime.datetime.now().strftime('%Y%m%d%H%M')        
+        DataMap.rs.sadd(self.rskeys['redis.datastore.key.option_implv_ts_set'], tk)
+        DataMap.rs.set(tk, json.dumps(matrix))
+        logging.debug("createImplVolChartData: saving option implvol matrix key=[%s] " % tk)
+        
+#       Sample values generated after executing the below:
+#
+#         ["strike",'P-20150828', 'C-20150828', 'P-20150929', 'C-20150929', ],[22400,0.285061765463,null,0.237686059088,null,],
+#         [22600,0.27409953156,null,0.236034059533,0.232365044818,],
+#         [22800,0.263230478241,0.320792426398,0.236005003448,0.234991796766,],
+#         [23000,0.251675376297,null,0.228237217842,0.230063445797,],
+#         [23200,0.241904579328,0.311715835907,0.2242344851,0.226293660467,],
+#         [23400,0.227987380777,0.284433386637,0.218600527288,0.203082047571,],
+        
+#         num_months = len(matrix[strike])
+#         s = '["strike",'
+#         for i in range(num_months):          
+#             s = s + "'P-%s', 'C-%s', " % (matrix[strike].keys()[i], matrix[strike].keys()[i])
+#         s = s + '],'
+#         for strike, items in sorted(matrix.iteritems()):
+#             s = s + '[%s,' % str(strike)
+#             l = ''
+#             for month, cp in sorted(items.iteritems()):
+#                 l = l + ''.join('%s,%s,' % (cp['P'], cp['C']))
+#                 #(('0.4f') % (cp['P']) if self.isfloat(cp['P']) else cp['P'], 
+#                 #                            ('0.4f') % (cp['C']) if self.isfloat(cp['C']) else cp['C'])
+#                                 
+#             s = s + l + '],\n'
+#         print s               
+
+    def isfloat(self, value):
+        try:
+          float(value)
+          return True
+        except ValueError:
+          return False
+
+            
+    def printContractByID(self, tick_id):
+        tick_id = int(tick_id)
+
+        if DataMap().isKeyInMap(tick_id):
+    
+            
+            s = 'Tick:%s [%s]>>' % (tick_id, ContractHelper.printContract(DataMap().get(tick_id)['contract'])) 
+            for field, value in DataMap().get(tick_id).iteritems():
+                if type(value) is not Contract:
+                    s = s + "%d:[%s], " % (field, value)
+            print "%s" % s
+
+        print 'values stored in redis:'
+        print "%s" % DataMap.rs.get(ContractHelper.makeRedisKey(DataMap().get(tick_id)['contract']))
+
+
+    def refreshRedisImplVol(self, sec):
+        while 1:
+            now = int(time.strftime('%H%M'))
+            
+            #print "%d" % now  + ','.join("%s,%s"% (v[0],v[1]) for k,v in DataMap.mkt_sessions.iteritems())
+            
+            if (now >= int(DataMap.mkt_sessions['morning'][0]) and now <= int(DataMap.mkt_sessions['morning'][1]) \
+                or  now >= int(DataMap.mkt_sessions['afternoon'][0]) and now <= int(DataMap.mkt_sessions['afternoon'][1])):
+                DataMap().createImplVolChartData()
+                logging.info('refreshRedisImplVol: update redis map every %ds' % sec)
+            
+            sleep(sec)
+
+    def refresh_portfolio(self, sec):
+        p = portfolio.PortfolioManager(self.config)
+        p.retrieve_position()
+#20150914
+        #p.recal_port()
+        cnt = 0
+        while 1:
+            now = int(time.strftime('%H%M'))
+            if (now >= int(DataMap.mkt_sessions['morning'][0]) and now <= int(DataMap.mkt_sessions['morning'][1]) \
+                or  now >= int(DataMap.mkt_sessions['afternoon'][0]) and now <= int(DataMap.mkt_sessions['afternoon'][1])):             
+                    s = p.recal_port()
+                    #print s
+                    
+            sleep(sec)
+
+
+
+def console(config, omd):
+    undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+    done = False
+    while not done:
+#        try:
+        print "Available commands are: l - list all subscribed contracts, i - force recal impl vol, s <H/M> <X> - H for HSI M for MHI manual subscription of HKFE options"
+        print "                        p - print portfolio summary"
+        print "                       <id> - list subscribed contract by id, q - terminate program"
+        cmd = raw_input(">>")
+        input = cmd.split(' ')
+        if input[0]  == "q":
+            done = True
+        elif input[0] == 'l' or input[0] == '':
+            #DataMap().printMap()
+            DataMap().printMapCSV()
+            DataMap().createOptionsPanelData()
+        elif input[0] == 'i':
+            DataMap().createImplVolChartData()
+        elif input[0] == 's':
+            symbol = 'HSI' if input[1].upper() == 'H' else 'MHI'
+            print symbol
+            X = int(input[2])
+            contractTuple = (symbol, 'FUT', 'HKFE', 'HKD', '', 0, '')
+            #add_subscription(self, contractTuple, month, X, right):
+            rc = map(lambda x: (omd.add_subscription(x[0], x[1], x[2], x[3])),\
+                                 [(contractTuple, m[0], X, 'P') for m in undly_months_prices] +\
+                                 [(contractTuple, m[0], X, 'C') for m in undly_months_prices])
+            print 'subscribed items: %s' % rc
+        elif input[0] == 'p':
+            port_key  = '%s_%s' % (config.get("redis", "redis.datastore.key.port_prefix").strip('"').strip("'"), \
+                            config.get("redis", "redis.datastore.key.port_summary").strip('"').strip("'"))
+
+            print 'position summary'
+            print '\n'.join('%s:\t\t%s' % (k,v) for k,v in sorted(json.loads(DataMap.rs.get(port_key)).iteritems()))
+            print '-----end position summary\nl'
+        elif isinstance(input[0], int): 
+            DataMap().printContractByID(input[0])
+        else:
+            pass
+            
+#         except:
+#             exc_type, exc_value, exc_traceback = sys.exc_info()
+#             traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
+
+
+def add_portfolio_subscription(config, omd):
+    logging.info('add_portfolio_subscription: subscribe market data for portfolio items')
+    p = portfolio.PortfolioManager(config)
+    p.retrieve_position()
+    
+    
+    undly_months_prices = eval(config.get("market", "option.underlying.month_price").strip('"').strip("'"))
+    
+    # example: MHI-20150929-22600-P
+    toks = map(lambda x: x.split('-'), p.get_pos_contracts())
+    
+    #[['MHI','20150929', 22600, 'P'], [...]]
+    for t in toks:
+        contractTuple = (t[0], 'FUT', 'HKFE', 'HKD', '', 0, '')
+        
+        #add_subscription(self, contractTuple, month, X, right):
+        rc = map(lambda x: (omd.add_subscription(x[0], x[1], x[2], x[3])),\
+                             [(contractTuple, m[0], t[2], 'P') for m in undly_months_prices] +\
+                             [(contractTuple, m[0], t[2], 'C') for m in undly_months_prices])        
+#         rc = map(lambda x: (omd.add_subscription(x[0], x[1], x[2], x[3])),\
+#                         [((tok[0], 'OPT', 'HKFE', 'HKD', '', 0, ''), m[0], tok[2], tok[3]) for tok in toks])
+                             
+    logging.info('add_portfolio_subscription: add results: %s' % rc)
+
+
+
+if __name__ == '__main__':
+        
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]    
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0: 
+        raise ValueError, "Failed to open config file" 
+      
+    logconfig = eval(config.get("options_data", "options_data.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)        
+    
+    DataMap().init_redis(config)
+    mdm = MarketDataManager(config)  
+    mdm.connect()
+    o = OptionsMarketDataManager(config, mdm)
+#     contractTuple = ('HSI', 'FUT', 'HKFE', 'HKD', '', 0, '')
+#     o.option_range(contractTuple, [['20150828', 22817.0], ['20150929', 22715.0]], 200, 0.08)
+
+    add_portfolio_subscription(config, o)
+
+    thread.start_new_thread(DataMap().refreshRedisImplVol, (60,))
+    thread.start_new_thread(DataMap().refresh_portfolio, (5,))
+    console(config, o)
+  
+
+       
+    mdm.disconnect()
+    
+         
+    

finopt/portfolio.py

@@ -0,0 +1,552 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import thread, threading
+from threading import Lock
+import ConfigParser
+from ib.ext.Contract import Contract
+from ib.opt import ibConnection, message
+from time import sleep
+import time, datetime
+import optcal
+import opt_serve
+import cherrypy
+import redis
+
+#from options_data import ContractHelper
+import options_data
+# Tick Value      Description
+# 5001            impl vol
+# 5002            delta
+# 5003            gamma
+# 5004            theta
+# 5005            vega
+# 5006            premium
+
+# 6001            avgCost
+# 6002            pos
+# 6003            totCost
+# 6004            avgPx
+# 6005            pos delta
+# 6006            pos theta
+# 6007            multiplier
+# 6020            pos value impact +1% vol change
+# 6021            pos value impact -1% vol change
+
+
+# IB tick types code reference
+# https://www.interactivebrokers.com/en/software/api/api.htm
+                    
+
+class PortfolioManager():
+    
+    config = {}
+    con = None
+    r_conn = None
+    port = []
+    grouped_options = None
+    download_state = 'undone'
+    quit = False
+    interested_types = ['OPT', 'FUT']
+    rs_port_keys = {}
+    ib_port_msg = []
+    tlock = None
+    
+    def __init__(self, config):
+        self.config = config
+#        config = ConfigParser.ConfigParser()
+#        config.read("config/app.cfg")
+        host = config.get("market", "ib.gateway").strip('"').strip("'")
+        port = int(config.get("market", "ib.port"))
+        appid = int(config.get("market", "ib.appid.portfolio"))   
+        self.rs_port_keys['port_conid_set'] = config.get("redis", "redis.datastore.key.port_conid_set").strip('"').strip("'")
+        self.rs_port_keys['port_prefix'] = config.get("redis", "redis.datastore.key.port_prefix").strip('"').strip("'")        
+        self.rs_port_keys['port_summary'] = config.get("redis", "redis.datastore.key.port_summary").strip('"').strip("'")
+        
+        
+        r_host = config.get("redis", "redis.server").strip('"').strip("'")
+        r_port = config.get("redis", "redis.port")
+        r_db = config.get("redis", "redis.db")             
+        
+        self.r_conn = redis.Redis(r_host, r_port, r_db)
+        
+        self.con = ibConnection(host, port, appid)
+        self.con.registerAll(self.on_ib_message)
+        #self.con.registerAll(self.xyz)
+        
+        
+        self.tlock = Lock()
+    
+    def retrieve_position(self):
+        self.connect()
+        
+        # clear previous saved values
+        self.port = []
+        self.ib_port_msg = []
+        self.clear_redis_portfolio()
+        
+        self.subscribe()
+        while not self.quit:
+            if self.download_state == 'done':
+                self.quit = True
+                
+    def clear_redis_portfolio(self):
+        l = map(lambda x: (self.r_conn.delete(x)), self.r_conn.keys(pattern='%s*' % (self.rs_port_keys['port_prefix'])))
+        #self.r_set(self.rs_port_keys['port_summary'], '{"Portfolio Retrieval": "Calculation in progress...Try again later!" }')
+        self.r_conn.set(self.rs_port_keys['port_summary'], '{"Portfolio Retrieval": "Calculation in progress...Try again later!" }')
+
+        logging.debug('clear_redis_portfolio: num items cleared: %d'% len(l))
+        
+    
+    def subscribe(self):
+
+            self.con.reqPositions()
+            #self.con.reqAccountSummary(100,'All', 'EquityWithLoanValue,NetLiquidation')
+
+            #self.con.register(self.on_ib_message, 'UpdateAccountValue')
+
+    def on_ib_message(self, msg):
+        
+        
+        
+        if msg.typeName in "position":
+            if self.download_state == 'undone':
+                logging.debug("%s" %  (options_data.ContractHelper.printContract(msg.contract)))
+                if msg.contract.m_secType in self.interested_types:
+                    logging.debug("PortfolioManager: getting position...%s" % msg)
+                    self.ib_port_msg.append(msg)
+                    #self.construct_port(msg)
+            
+        if msg.typeName == 'positionEnd':
+            
+
+            for pm in self.ib_port_msg:
+                self.construct_port(pm)
+                
+            self.recal_port()
+            self.group_pos_by_strike()
+            logging.debug("PortfolioManager: Complete position download. Disconnecting...")
+            self.disconnect()
+            
+
+
+            self.download_state = "done"
+            
+    def group_pos_by_strike(self):
+
+
+        # split into lines of position       
+        m = map(lambda x: x.split(','), self.port)
+        # transform each line into two elements. first one is a key created 
+        # by combining right and strike, the second is the product of 
+        # position * conversion ratio (HSI=50, MSI=10)
+        n = map(lambda x: (x[3] + x[4], float(x[5]) * float(x[6])/50), m)
+        
+        #p = dict(set(map(lambda x:(x[0], 0), n)))
+        
+        def sumByKey(f, n):
+            # filter the list with only unique keys
+            # transform the list into a dict
+            p = dict(set(map(f, n)))
+            # add the numbers together on key
+            l =[]
+            for x in n:
+                p[x[0]] += x[1]
+                
+            return [(k[1:], k[0:1],v) for k,v in p.iteritems()]
+         
+         
+           
+        #print len(n),n
+        # initialize a list of strikes and the position sum to 0
+        # pass the list to sumByKey routine
+        self.grouped_options = sumByKey(lambda x:(x[0], 0), n)
+        #print len(l), sorted(l)
+        
+    def group_pos_by_right(self):        
+    # group by put, call (summing up all contracts by right type,ignoring strikes)
+        # split into lines of position       
+        m = map(lambda x: x.split(','), self.port)
+        q = map(lambda x: (x[3], float(x[5]) * float(x[6])/50), m)
+        u = dict(set(map(lambda x:(x[0], 0.0), q)))
+#        print u, q
+        for x in q:
+            u[x[0]] += x[1] 
+        return [(a,b) for a,b in u.iteritems()]
+    
+    def get_grouped_options_str_array(self):
+        s = ''
+        for e in sorted(self.grouped_options):
+            s += "[%f,%s,%s]," % (float(e[0])/100.0, e[2] if e[1] == 'P' else 0, e[2] if e[1] == 'C' else 0)
+        return s
+    
+    
+    def get_traded_months(self):
+        
+        li = []
+        for l in self.port:
+            toks=  l.split(',')
+            li.append(toks[2])
+        #print sorted(list(set(li)))
+        return sorted(list(set(li)))
+    
+#     def get_tbl_pos_csv(self):
+#         s_cols = [0,1,2,3,4]
+#         i_cols = [5,6,7]
+#         s = '["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],'
+#         
+#         for l in sorted(self.port):
+#             content = ''    
+#             toks= l.split(',')
+#  #           print toks
+#             for i in s_cols:
+#                 content += '"%s",' % toks[i]
+#             for i in i_cols:
+#                 content += '%s,' % toks[i]
+#             
+#                 
+#             s += "[%s]," % content
+#         return s       
+
+
+    def get_greeks(self, c_key):
+        #print c_key, len(c_key), self.r_conn.get('a')
+        
+        gdata = self.r_conn.get(c_key)
+        
+        #print gdata
+        gmap = json.loads(gdata) if gdata <> None else None
+        return gmap
+
+
+    def get_pos_contracts(self):
+        s_cols = [0,2,4,3]
+        cs = []
+        for l in sorted(self.port):
+            content = ''    
+            toks= l.split(',')
+            c_key = '-'.join('%s' % toks[i] for i in s_cols)
+            cs.append(c_key)
+        return cs
+
+    def get_tbl_pos_csv(self):
+
+        # WARNING: this routine will crash if a new instrument is traded during the day
+        # but it is not previously been subscribed 
+        # the gmap will be missing from redis
+        # the workaround is to restart options_data.py 
+        # to refresh all positions and add back any
+        # new instruments to its subscription list 
+
+        pall = set(self.r_conn.keys(pattern='%s*' % self.rs_port_keys['port_prefix']))
+        s = '["symbol","right","avgcost","spotpx","pos","delta","theta","pos_delta","pos_theta","unreal_pl","last_updated"],'
+        
+        def split_toks(x):
+            pmap = json.loads(self.r_conn.get(x))
+            #print pmap
+            gmap = json.loads(self.r_conn.get(x[3:]))
+            #print gmap
+            s = '["%s","%s",%f,%f,%f,%f,%f,%f,%f,%f,"%s"],' % (x[3:], x[len(x)-1:], pmap['6001'], gmap['5006'], pmap['6002'],\
+                                                                         gmap['5002'],gmap['5004'],\
+                                                                         pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated'])
+            return s                                                          
+            
+        end_s = s + ''.join (split_toks( x ) for x in pall)
+        return end_s 
+
+
+    def get_tbl_pos_csv_old(self, calGreeks=False):
+        s_cols = [0,1,2,3,4]
+        i_cols = [5,6,7]
+        s = '["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],'
+        
+        for l in sorted(self.port):
+            content = ''    
+            toks= l.split(',')
+ #           print toks
+            for i in s_cols:
+                content += '"%s",' % toks[i]
+            for i in i_cols:
+                content += '%s,' % toks[i]
+            
+                
+            s += "[%s]," % content
+        return s  
+
+    def get_portfolio_summary(self):
+
+        
+        return json.loads(self.r_conn.get(self.rs_port_keys['port_summary']))
+    
+
+    def recal_port(self):
+        # http://stackoverflow.com/questions/448034/multithreaded-resource-access-where-do-i-put-my-locks
+        
+        # a wrapper just to ensure only one thread is doing the portfolio recalculation
+        logging.debug('recal_port: acquiring lock...%s' % threading.currentThread())
+        self.tlock.acquire()
+        try:
+            s = self.recal_port_rentrant_unsafe()
+        finally:
+            logging.debug('recal_port: completed recal. releasing lock...')
+            self.tlock.release()  
+            
+        return s
+
+    def recal_port_rentrant_unsafe(self):
+        
+        
+        logging.debug('PortfolioManager: recal_port')
+
+        # retrieve the portfolio entries from redis, strip the prefix in front
+        plines = map(lambda k: (k.replace(self.rs_port_keys['port_prefix'] + '_', '')),\
+                                 self.r_conn.keys(pattern=('%s*'% self.rs_port_keys['port_prefix'])))
+                                 
+        logging.info ("PortfolioManager: recal_port-> gathering position entries from redis %s" % plines)
+        
+        pos_summary = {'delta_c': 0.0, 'delta_p': 0.0, 'delta_all': 0.0,\
+                       'theta_c': 0.0, 'theta_p': 0.0, 'theta_all': 0.0,\
+                       'delta_1percent' : 0.0, 'theta_1percent' : 0.0,\
+                       'iv_plus1p': 0.0, 'iv_minus1p': 0.0, 'unreal_pl': 0.0}
+        l_gmap = []
+        l_skipped_pos =[]       
+        t_pos_multiplier = 0.0
+        
+        for ckey in plines:
+        
+            gmap = self.get_greeks(ckey)
+            logging.debug('PortfolioManager: recal_port greeks market data %s->%s ' % (ckey, gmap))
+            logging.debug('PortfolioManager: recal_port position-map->%s' % (self.r_get(ckey)))
+            pmap = json.loads(self.r_get(ckey)) 
+            
+            
+            
+            if gmap:
+            
+            # Tick Value      Description
+            # 5001            impl vol
+            # 5002            delta
+            # 5003            gamma
+            # 5004            theta
+            # 5005            vega
+            # 5006            premium        
+            # 6001            avgCost
+            # 6002            pos
+            # 6003            totCost
+            # 6004            avgPx
+            # 6005            pos delta
+            # 6006            pos theta
+            # 6007            multiplier
+            # 6009            curr_port_value
+            # 6008            unreal_pl
+            # 6020            pos value impact +1% vol change
+            # 6021            pos value impact -1% vol change
+            
+           
+                def pos_delta():                 
+                    pd = pmap['6002'] * gmap['5002'] * pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_delta: %f' % pd)
+                    return pd
+                
+                def pos_theta():
+                    pd = pmap['6002'] * gmap['5004'] * pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_theta: %f' % pd)
+                    return pd
+ 
+                def pos_avg_px():
+                    pd = pmap['6001'] / pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_avg_px: %f' % pd)
+                    return pd                   
+                    
+                def pos_tot_cost():
+                    pd = pmap['6001'] * pmap['6002'] * pmap['6007']
+                    logging.debug('PortfolioManager: recal_port: pos_tot_cost: %f' % pd)
+                    return pd                   
+                
+                def pos_unreal_pl():
+                    #(spot premium * multiplier - avgcost) * pos) 
+                    v = (gmap['5006'] * pmap['6007'] - pmap['6001']) * pmap['6002'] 
+                    return v 
+    
+                
+    
+                #logging.debug('PortfolioManager: recal_port greeks %f' % pos_delta(gmap))
+                
+                pmap['6005'] = pos_delta()
+                pmap['6006'] = pos_theta()
+                pmap['6004'] = pos_avg_px()
+                pmap['6003'] = pos_tot_cost()
+                pmap['6008'] = pos_unreal_pl()
+                pmap['last_updated'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+                
+                l_gmap.append(pmap)          
+
+                #t_pos_multiplier += (pmap['6002'] * pmap['6007'])
+
+                right = ckey.split('-')[3].lower()
+                pos_summary['delta_' + right] += pmap['6005']
+                pos_summary['delta_all'] += pmap['6005']
+                pos_summary['theta_' + right] += pmap['6006']
+                pos_summary['theta_all'] += pmap['6006']
+                pos_summary['unreal_pl'] += pmap['6008']
+                pos_summary['delta_1percent'] += (pos_summary['delta_all'] / (pmap['6002'] * pmap['6007']))
+                # delta_1% = pos_ / (pos * multiplier)
+                
+                
+                #print 'con,right,avgcost,spot px,pos,delta,theta, pos_delta,pos_theta,pos_unreal_pl,last_updated'
+#                 print ( 'PT_entries: %s,%s,%f,%f,%f,%f,%f,%f,%f,%f,%s' % (ckey, right, pmap['6001'], gmap['5006'], pmap['6002'],\
+#                                                                      gmap['5002'],gmap['5004'],\
+#                                                                      pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated']
+#                                                                      ))
+#20150911                
+                #print pmap
+                #print json.dumps(pmap)
+                self.r_set(ckey, json.dumps(pmap))
+                logging.debug('PortfolioManager: update position in redis %s' % self.r_get(ckey))
+                
+            else:
+                l_skipped_pos.append(ckey)
+
+            
+#            self.r_set(ckey, json.dumps(pmap))
+#            logging.debug('PortfolioManager: update position in redis %s' % self.r_get(ckey))
+
+ 
+        #pos_summary['delta_1percent'] = (pos_summary['delta_all'] / t_pos_multiplier)
+        
+        
+
+ 
+        if len(l_skipped_pos) > 0:
+            logging.warn('***************** PortfolioManager: recal_port. SOME POSITIONS WERE NOT PROCESSED!')
+            logging.warn('----------------- DO NOT rely on the numbers in the Summary dictionary (pos_summary)')
+            logging.warn('----------------- Please check your portfolio through other means or subscribe missing')
+            logging.warn('----------------- market data in options_serve.py console. ')
+        logging.info('-------------- POSITION SUMMARY')
+        pos_summary['last_updated'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')    
+        pos_summary['entries_skipped'] = l_skipped_pos
+        pos_summary['status'] = 'OK' if len(l_skipped_pos) == 0 else 'NOT_OK'
+        #self.r_set(self.rs_port_keys['port_summary'], json.dumps(pos_summary) )
+        self.r_conn.set(self.rs_port_keys['port_summary'], json.dumps(pos_summary) )
+
+        logging.info(pos_summary)
+        
+        logging.warn('-------------- Entries for which the greeks are not computed!! %s' %\
+                        ','.join(' %s' % k for k in l_skipped_pos))
+        
+        return l_gmap
+ 
+
+    def construct_port(self, pos_msg):
+        # port structure
+        #
+        # exch,type,contract_mth,right, strike,con_ration,pos,avgcost
+        # 
+        # Tick Value      Description
+        # 5001            impl vol
+        # 5002            delta
+        # 5003            gamma
+        # 5004            theta
+        # 5005            vega
+        # 5005            premium
+        
+        # 6001            avgCost
+        # 6002            pos
+    
+        
+        toks = options_data.ContractHelper.printContract(pos_msg.contract).split('-')
+        s = ''
+        
+        slots = [0, 1, 4, 6]
+        s = s + '%s,%s,%s' % (','.join(toks[i] for i in slots), toks[5].replace('.0', ''), '50.0' if toks[0][1:] == 'HSI' else '10.0')
+        s = s.replace('[', '') + ",%0.4f,%0.4f" % (pos_msg.pos, pos_msg.avgCost)
+        
+        
+        self.port.append(s)
+                
+        ckey = options_data.ContractHelper.makeRedisKey(pos_msg.contract)
+        multiplier = 50.0 if toks[0][1:] == 'HSI' else 10.0
+
+        
+        
+        self.r_set(ckey, json.dumps({"contract": ckey, "6002": pos_msg.pos, "6001": pos_msg.avgCost, "6007": multiplier}))
+        #self.recal_port(ckey)
+ 
+    #the wrapper functions add a prefix session id to every key
+    #
+    def r_set(self, key, val):
+        return self.r_conn.set("%s_%s" % (self.rs_port_keys['port_prefix'], key), val)
+        
+    
+    def r_get(self, key):
+        return self.r_conn.get("%s_%s" % (self.rs_port_keys['port_prefix'], key))
+    
+    
+    def r_sadd(self, key, val):
+        return self.r_conn.sadd("%s_%s" % (self.rs_port_keys['port_prefix'], key), val)
+    
+    def r_sismember(self, set, val):
+        return self.r_conn.sismember("%s_%s" % (self.rs_port_keys['port_prefix'], set), val)
+
+    def r_smembers(self, key):
+        return self.r_conn.smembers("%s_%s" % (self.rs_port_keys['port_prefix'], key))
+       
+
+    def connect(self):
+        self.con.connect()
+
+        
+    def disconnect(self):
+
+        self.con.disconnect()
+        self.quit = True
+
+if __name__ == '__main__':
+           
+#     logging.basicConfig(#filename = "log/port.log", filemode = 'w', 
+#                         level=logging.DEBUG,
+#                         format='%(asctime)s %(levelname)-8s %(message)s')      
+#     
+#     config = ConfigParser.ConfigParser()
+#     config.read("config/app.cfg")
+#     p = PortfolioManager(config)
+#     p.retrieve_position()
+#     print p.get_portfolio_summary()
+#     print p.get_tbl_pos_csv()
+
+    if len(sys.argv) != 2:
+        print("Usage: %s <config file>" % sys.argv[0])
+        exit(-1)    
+
+    cfg_path= sys.argv[1:]    
+    config = ConfigParser.SafeConfigParser()
+    if len(config.read(cfg_path)) == 0:      
+        raise ValueError, "Failed to open config file" 
+    
+    logconfig = eval(config.get("portfolio", "portfolio.logconfig").strip('"').strip("'"))
+    logconfig['format'] = '%(asctime)s %(levelname)-8s %(message)s'    
+    logging.basicConfig(**logconfig)        
+
+    p = PortfolioManager(config)
+    p.retrieve_position()
+    print p.get_portfolio_summary()
+    print p.get_tbl_pos_csv()
+
+
+
+    #p.recal_port()
+#    print p.get_tbl_pos_csv()
+#     print p.get_grouped_options_str_array()
+#     print p.group_pos_by_right()
+
+    # sample ouput    
+# ["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],["HSI","OPT","20150828","C","22600",50.0,0.0000,0.0000,],["HSI","OPT","20150828","C","23000",50.0,-1.0000,1770.0000,],["HSI","OPT","20150828","C","23600",50.0,-2.0000,1470.0000,],["HSI","OPT","20150828","C","23800",50.0,-1.0000,920.0000,],["HSI","OPT","20150828","C","24000",50.0,-2.0000,1820.0000,],["HSI","OPT","20150828","C","24200",50.0,-1.0000,3120.0000,],["HSI","OPT","20150828","C","24800",50.0,-1.0000,220.0000,],["HSI","OPT","20150828","P","18000",50.0,-2.0000,1045.0000,],["HSI","OPT","20150828","P","18600",50.0,-1.0000,1120.0000,],["HSI","OPT","20150828","P","18800",50.0,-1.0000,1570.0000,],["HSI","OPT","20150828","P","19800",50.0,-1.0000,870.0000,],["HSI","OPT","20150828","P","20200",50.0,-1.0000,970.0000,],["HSI","OPT","20150828","P","20800",50.0,-2.0000,970.0000,],["HSI","OPT","20150828","P","21600",50.0,-1.0000,1570.0000,],["HSI","OPT","20150828","P","21800",50.0,-7.0000,1955.7143,],["HSI","OPT","20150828","P","23200",50.0,1.0000,25930.0000,],["HSI","OPT","20150929","C","24400",50.0,1.0000,24880.0000,],["HSI","OPT","20150929","P","21600",50.0,0.0000,0.0000,],["HSI","OPT","20150929","P","21800",50.0,2.0000,52713.3333,],["HSI","OPT","20150929","P","22600",50.0,3.0000,39763.3333,],["MHI","OPT","20150828","C","24400",10.0,-1.0000,2603.0000,],["MHI","OPT","20150828","P","20800",10.0,-1.0000,313.0000,],["MHI","OPT","20150828","P","21000",10.0,-1.0000,363.0000,],["MHI","OPT","20150828","P","23600",10.0,5.0000,4285.0000,],["MHI","OPT","20150929","C","24400",10.0,1.0000,4947.0000,],["MHI","OPT","20150929","P","21600",10.0,1.0000,12657.0000,],["MHI","OPT","20150929","P","22600",10.0,1.0000,9877.0000,],["MHI","OPT","20150929","P","23600",10.0,4.0000,7757.0000,],
+# [180.000000,-2.0,0],[186.000000,-1.0,0],[188.000000,-1.0,0],[198.000000,-1.0,0],[202.000000,-1.0,0],[208.000000,-2.2,0],[210.000000,-0.2,0],[216.000000,-0.8,0],[218.000000,-5.0,0],[226.000000,0,0.0],[226.000000,3.2,0],[230.000000,0,-1.0],[232.000000,1.0,0],[236.000000,0,-2.0],[236.000000,1.8,0],[238.000000,0,-1.0],[240.000000,0,-2.0],[242.000000,0,-1.0],[244.000000,0,1.0],[248.000000,0,-1.0],
+# {'P': 0.0, 'C': 0.0} [('P', 0.8), ('P', 0.0), ('C', -2.0), ('C', -2.0), ('P', -1.0), ('C', 0.2), ('P', -1.0), ('P', -0.2), ('C', 0.0), ('P', -0.2), ('C', -1.0), ('P', -1.0), ('C', -1.0), ('P', 1.0), ('P', 0.2), ('P', 2.0), ('C', 1.0), ('P', -2.0), ('P', -7.0), ('P', 1.0), ('P', -1.0), ('P', -1.0), ('P', 0.2), ('C', -1.0), ('P', 3.0), ('C', -1.0), ('C', -0.2), ('P', -2.0)]
+# [('P', -8.200000000000001), ('C', -7.0)]
+
+    
+         
+    

finopt/portfolio_ex.py

@@ -0,0 +1,471 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import thread
+import ConfigParser
+from ib.ext.Contract import Contract
+from ib.opt import ibConnection, message
+from time import sleep
+import time, datetime
+import optcal
+import opt_serve
+import cherrypy
+import redis
+from helper_func.py import dict2str, str2dict
+from options_data import ContractHelper 
+import uuid            
+
+# 
+# IB tick types code reference
+# https://www.interactivebrokers.com/en/software/api/api.htm
+                    
+                    
+#
+#
+# the program starts by connecting to IB to retrieve the current positions of the account
+# for each position line returned by IB API
+# the program derive the instrument underlying 
+# next it subscribes market data for the instrument and the underlying
+# finally it stores all the information into a map and also into the redis datastore
+#                     
+
+class PortfolioManagerEx():
+    
+    config = {}
+    con = None
+    r_conn = None
+    
+    #20150827 modifed the port data structure
+    # port = {'<conId>': {'undly:'id', 'instr_id': 'id', 'contract':<contract tuple>, 'pos': []}, 's_pos' = [<a list containing lines of positions separated by commas]}
+    # port = {'1234': 
+    #            {'undly':{'id': 444}}, 'contract': c, 'pos': []    },
+    #         '333322':
+    #            {'undly':{'id': 444}}, 'contract': c, 'pos': []    },
+    #         's_pos' : [
+    #
+    #                         ["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],["HSI","OPT","20150828","C","22600",50.0,0.0000,0.0000,],["HSI","OPT","20150828","C","23000",50.0,-1.0000,1770.0000,],["HSI","OPT","20150828","C","23600",50.0,-2.0000,1470.0000,],["HSI","OPT","20150828","C","23800",50.0,-1.0000,920.0000,],["HSI","OPT","20150828","C","24000",50.0,-2.0000,1820.0000,],["HSI","OPT","20150828","C","24200",50.0,-1.0000,3120.0000,],["HSI","OPT","20150828","C","24800",50.0,-1.0000,220.0000,],["HSI","OPT","20150828","P","18000",50.0,-2.0000,1045.0000,],["HSI","OPT","20150828","P","18600",50.0,-1.0000,1120.0000,],["HSI","OPT","20150828","P","18800",50.0,-1.0000,1570.0000,],["HSI","OPT","20150828","P","19800",50.0,-1.0000,870.0000,],["HSI","OPT","20150828","P","20200",50.0,-1.0000,970.0000,],["HSI","OPT","20150828","P","20800",50.0,-2.0000,970.0000,],["HSI","OPT","20150828","P","21600",50.0,-1.0000,1570.0000,],["HSI","OPT","20150828","P","21800",50.0,-7.0000,1955.7143,],["HSI","OPT","20150828","P","23200",50.0,1.0000,25930.0000,],["HSI","OPT","20150929","C","24400",50.0,1.0000,24880.0000,],["HSI","OPT","20150929","P","21600",50.0,0.0000,0.0000,],["HSI","OPT","20150929","P","21800",50.0,2.0000,52713.3333,],["HSI","OPT","20150929","P","22600",50.0,3.0000,39763.3333,],["MHI","OPT","20150828","C","24400",10.0,-1.0000,2603.0000,],["MHI","OPT","20150828","P","20800",10.0,-1.0000,313.0000,],["MHI","OPT","20150828","P","21000",10.0,-1.0000,363.0000,],["MHI","OPT","20150828","P","23600",10.0,5.0000,4285.0000,],["MHI","OPT","20150929","C","24400",10.0,1.0000,4947.0000,],["MHI","OPT","20150929","P","21600",10.0,1.0000,12657.0000,],["MHI","OPT","20150929","P","22600",10.0,1.0000,9877.0000,],["MHI","OPT","20150929","P","23600",10.0,4.0000,7757.0000,],
+    #                         [180.000000,-2.0,0],[186.000000,-1.0,0],[188.000000,-1.0,0],[198.000000,-1.0,0],[202.000000,-1.0,0],[208.000000,-2.2,0],[210.000000,-0.2,0],[216.000000,-0.8,0],[218.000000,-5.0,0],[226.000000,0,0.0],[226.000000,3.2,0],[230.000000,0,-1.0],[232.000000,1.0,0],[236.000000,0,-2.0],[236.000000,1.8,0],[238.000000,0,-1.0],[240.000000,0,-2.0],[242.000000,0,-1.0],[244.000000,0,1.0],[248.000000,0,-1.0],
+    #                    ]
+    #    
+    #        }
+    port = {}
+    #port = []
+    grouped_options = None
+    download_state = 'undone'
+    quit = False
+    interested_types = ['OPT', 'FUT']
+    EXCHANGE = 'HKFE'
+    cal_greeks_settings = {}
+    rs_port_keys = {}
+    # undlys = [(con_id, contract),...]
+    undlys = []
+    curr_con_id = 0
+    session_id = None
+    
+    def __init__(self, config):
+        self.config = config
+        config = ConfigParser.ConfigParser()
+        config.read("config/app.cfg")
+        host = config.get("market", "ib.gateway").strip('"').strip("'")
+        port = int(config.get("market", "ib.port"))
+        appid = int(config.get("market", "ib.appid.portfolio"))   
+        
+        r_host = config.get("redis", "redis.server").strip('"').strip("'")
+        r_port = config.get("redis", "redis.port")
+        r_db = config.get("redis", "redis.db")             
+        
+        
+        self.r_conn = redis.Redis(r_host, r_port, r_db)
+        
+        self.con = ibConnection(host, port, appid)
+        self.con.registerAll(self.on_ib_message)
+        #self.con.registerAll(self.xyz)
+
+        self.cal_greeks_settings['rate'] = config.get("market", "portfolio.cal.param.rate")
+        self.cal_greeks_settings['div'] = config.get("market", "portfolio.cal.param.div")
+        self.rs_port_keys['port_conid_set'] = config.get("redis", "redis.datastore.key.port_conid_set").strip('"').strip("'")
+        
+        
+        self.session_id = self.gen_session_id()
+        logging.info("********** Starting up portfolio-ex manager")
+        logging.info("********** session id preix is [%s], this id is prepended to every key in the redis db" % self.session_id)
+        logging.info("**********")
+        
+        logging.info("********** clear the redis datastore for all records that start with this session id [%s]" % self.session_id)
+        self.clear_redis_portfolio()
+        
+    
+    
+    def clear_redis_portfolio(self):
+        map(lambda x: (self.r_conn.delete(x)), self.r_conn.keys(pattern='%s*'%(self.session_id)))
+    
+    def gen_session_id(self):
+        #return uuid.uuid4()
+        return 'pt'
+    
+    def retrieve_position(self):
+        self.connect()
+        self.port['s_pos'] = []
+        self.con.reqPositions()
+
+
+    def start(self):
+        self.retrieve_position()
+        while self.download_state <> 'done':
+            pass
+            
+        def endless_loop():
+            while not self.quit:
+                self.console()
+                
+            # unsubscribe instruments
+            sys.exit(0)
+        
+         
+        thread.start_new_thread(endless_loop(), )
+                
+    def stop(self):
+        self.quit = True
+
+
+
+    def console(self):
+        #try:
+        print "Available commands are: t - pos in csv, k - port key maps, r - group position by right, j - dump json string, q - terminate program"
+        cmd = raw_input(">>")
+        input = cmd.split(' ')
+        if input[0]  == "q":
+            self.quit = True
+        elif input[0] == 't':
+            print self.get_tbl_pos_csv()
+        elif input[0] == 'k':
+            print self.print_portfolio_key_maps()
+        elif input[0] == 'r':
+            print self.group_pos_by_right()
+        elif input[0] == 'j':        
+            print json.dumps(self.port)
+        else:
+            pass
+        #except:
+            
+            #exc_type, exc_value, exc_traceback = sys.exc_info()
+            #traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
+        
+    def subscribe(self, contract):
+
+            #self.con.reqPositions()
+            #self.con.reqAccountSummary(100,'All', 'EquityWithLoanValue,NetLiquidation')
+            #self.con.register(self.on_ib_message, 'UpdateAccountValue')
+        id = -1
+        if self.isContractSubscribed(self.curr_con_id) == False:
+            #logging.debug( 'MarketDataManager:subscribe subscrbing to' + ContractHelper.printContract(contract))
+            id = self.curr_con_id
+            self.con.reqMktData(self.curr_con_id, contract, '', False)
+            #self.r_conn.sadd(self.rs_port_keys['port_conid_set'], self.curr_con_id)
+            #self.r_sadd(self.rs_port_keys['port_conid_set'], self.curr_con_id)
+            #self.r_conn.set(self.curr_con_id, {})
+            #self.r_set(self.curr_con_id, {})
+            
+            
+            logging.debug( 'PortfolioManagerEx:subscribe >> Add record into redis set rs_port_keys["port_conid_set"]- >%s_%s, contract=>%s' %\
+                                                         (self.session_id, str(self.curr_con_id), ContractHelper.printContract(contract)))
+            self.curr_con_id = self.curr_con_id + 1
+            
+        else:
+            logging.debug("PortfolioManagerEx: Contract already subscribed %s" % ContractHelper.printContract(contract))
+        return id
+
+
+    #the wrapper functions add a prefix session id to every key
+    #
+    def r_set(self, key, val):
+        return self.r_conn.set("%s_%s" % (self.session_id, key), val)
+        
+    
+    def r_get(self, key):
+        return self.r_conn.get("%s_%s" % (self.session_id, key))
+    
+    
+    def r_sadd(self, key, val):
+        return self.r_conn.sadd("%s_%s" % (self.session_id, key), val)
+    
+    def r_sismember(self, set, val):
+        return self.r_conn.sismember("%s_%s" % (self.session_id, set), val)
+
+    def r_smembers(self, key):
+        return self.r_conn.smembers("%s_%s" % (self.session_id, key))
+
+    def isContractSubscribed(self, id):
+        
+        #logging.debug("PortfolioManagerEx: isContractSubscribed %s" % ('True' if self.r_conn.sismember(self.rs_port_keys['port_conid_set'], id) else 'False')) 
+        logging.debug("PortfolioManagerEx: isContractSubscribed %s" % ('True' if self.r_sismember(self.rs_port_keys['port_conid_set'], id) else 'False'))
+        return self.r_sismember(self.rs_port_keys['port_conid_set'], id)
+
+
+    def link_undly_to_instrument(self, pos_contract):
+        
+        # determine the fut contract name
+        # check the underlying id list
+        # if the contract is not subscribed yet, subscribe for mkt data and assign a new con id
+        # link the underlying id to the contract
+        # 
+        
+        
+        
+        # transform the pos_contract to its underlying secType
+        undly = Contract()
+        undly.m_symbol = pos_contract.m_symbol
+        undly.m_expiry = pos_contract.m_expiry
+        undly.m_exchange = PortfolioManagerEx.EXCHANGE
+        undly.m_secType = 'FUT'
+        undly.m_right = ''
+        undly.m_strike = 0
+        undly.m_conId = ''
+        
+        def equals(u1, u2):
+            return True if u1.m_exchange == u2.m_exchange and \
+               u1.m_expiry == u2.m_expiry and \
+               u1.m_symbol == u2.m_symbol else False
+           
+        logging.debug("PortfolioManagerEx: link_undly_to_instrument > transform deriv to its underling instrument %s" \
+                      % ContractHelper.printContract(pos_contract)) 
+        
+        for u in self.undlys:
+            if equals(u[1], undly):
+                # return the undelying con_id
+                return u[0]
+
+        logging.debug("PortfolioManagerEx: underlying is not subscribed yet. calling self.subscribe %s" \
+                      % ContractHelper.printContract(undly)) 
+        # the undelying is not in undlys
+        id  = self.subscribe(undly)
+        logging.debug("PortfolioManagerEx: underlying tick id is %d, adding the entry into self.undlys list (id, undly)" % id)
+        # remember this id
+        self.undlys.append((id, undly))
+                
+        return id       
+            
+            
+        
+    def construct_port(self, pos_msg):
+
+        
+        def split_contract(contract):
+        # port line for google api and cherrypy consumption 
+        #
+        # exch,type,contract_mth,right, strike,con_ration,pos,avgcost
+        #             
+            toks = ContractHelper.printContract(contract).split('-')
+            s = ''
+            slots = [0, 1, 4, 6]
+            s = s + '%s,%s,%s' % (','.join(toks[i] for i in slots), toks[5].replace('.0', ''), '50.0' if toks[0][1:] == 'HSI' else '10.0')
+            s = s.replace('[', '')
+            return s
+        
+        
+        
+        
+        self.port[pos_msg.contract.m_conId]={}
+        self.port[pos_msg.contract.m_conId]['contract'] = ContractHelper.convert2Tuple(pos_msg.contract)
+        s = split_contract(pos_msg.contract) + ",%0.4f,%0.4f" % (pos_msg.pos, pos_msg.avgCost)
+        self.port['s_pos'].append(s)
+        
+        # pos keys: imvol,delta,gamma,theta,vega,npv,avgcost,pos,spotp        
+        keys = ['imvol','delta','gamma','theta','vega','npv','spotp']
+        for k in keys:
+            self.port[pos_msg.contract.m_conId][k] = 0.0
+        self.port[pos_msg.contract.m_conId]['avgcost'] = pos_msg.avgCost
+        self.port[pos_msg.contract.m_conId]['pos'] = pos_msg.pos
+        
+
+        undlyid = self.link_undly_to_instrument(pos_msg.contract)
+        tickid = self.subscribe(pos_msg.contract)
+        logging.debug("construct_port - portfolio id %d, contract id %d, link underlying id %d" % (pos_msg.contract.m_conId, tickid, undlyid))
+        self.port[pos_msg.contract.m_conId]['undly'] = undlyid
+        self.port[pos_msg.contract.m_conId]['instr_id'] = tickid
+        
+        
+        self.r_sadd(self.rs_port_keys['port_conid_set'], pos_msg.contract.m_conId)
+        jstr = json.dumps(self.port[pos_msg.contract.m_conId])
+        self.r_set(pos_msg.contract.m_conId, jstr) 
+        logging.debug('construct_port: id->%d dump->%s' % (pos_msg.contract.m_conId, jstr))
+        
+        
+        def add_item(k, v):            
+            if not self.r_get(k):
+                self.r_set(k, [v])
+            else:
+                l = eval(self.r_get(k))
+                l.append(v)
+                self.r_set(k, l)
+        
+        # create reverse lookup map: instr tick id -> pos_con_id
+        add_item(tickid, pos_msg.contract.m_conId)        
+        add_item(undlyid, tickid)
+            
+            
+            
+
+    
+
+    def on_ib_message(self, msg):
+        
+        #print msg.typeName, msg
+        
+        if msg.typeName in "position":
+            if self.download_state == 'undone':
+                
+                logging.debug("on_ib_message: %s %s %0.4f,%0.4f" %  (msg.account, ContractHelper.printContract(msg.contract), msg.pos, msg.avgCost))
+                if msg.contract.m_secType in self.interested_types:
+                    
+                    
+                    
+                    self.construct_port(msg)
+
+
+            
+        if msg.typeName == 'positionEnd':
+            self.download_state = "done"
+            logging.debug("on_ib_message: complete downloading positions")
+            #logging.info('-------')
+            #logging.info(json.dumps(self.port))
+
+        
+
+        if msg.typeName == 'tickPrice':
+            id = msg.field
+            print 'tick change for %d' % id
+            for u in self.undlys:
+                if u[0] == id:
+                    print 'recal all pos affected by %d' % u[0]
+                    return 
+        
+            print 'recal pos for '
+            # worker_recal([port_ids...], price)
+    
+    
+    
+            
+    def group_pos_by_strike(self):
+
+
+        # split into lines of position       
+        m = map(lambda x: x.split(','), self.port['s_pos'])
+        # transform each line into two elements. first one is a key created 
+        # by combining right and strike, the second is the product of 
+        # position * conversion ratio (HSI=50, MSI=10)
+        n = map(lambda x: (x[3] + x[4], float(x[5]) * float(x[6])/50), m)
+        
+        #p = dict(set(map(lambda x:(x[0], 0), n)))
+        
+        def sumByKey(f, n):
+            # filter the list with only unique keys
+            # transform the list into a dict
+            p = dict(set(map(f, n)))
+            # add the numbers together on key
+            l =[]
+            for x in n:
+                p[x[0]] += x[1]
+                
+            return [(k[1:], k[0:1],v) for k,v in p.iteritems()]
+         
+         
+           
+        #print len(n),n
+        # initialize a list of strikes and the position sum to 0
+        # pass the list to sumByKey routine
+        self.grouped_options = sumByKey(lambda x:(x[0], 0), n)
+        #print len(l), sorted(l)
+        
+    def group_pos_by_right(self):        
+    # group by put, call (summing up all contracts by right type,ignoring strikes)
+        # split into lines of position       
+        m = map(lambda x: x.split(','), self.port['s_pos'])
+        q = map(lambda x: (x[3], float(x[5]) * float(x[6])/50), m)
+        u = dict(set(map(lambda x:(x[0], 0.0), q)))
+        print u, q
+        for x in q:
+            u[x[0]] += x[1] 
+        return [(a,b) for a,b in u.iteritems()]
+    
+    def get_grouped_options_str_array(self):
+        s = ''
+        for e in sorted(self.grouped_options):
+            s += "[%f,%s,%s]," % (float(e[0])/100.0, e[2] if e[1] == 'P' else 0, e[2] if e[1] == 'C' else 0)
+        return s
+    
+    
+    def print_portfolio_key_maps(self):
+        print ("port id, instr id, undly id -> [link instr id]")
+        for pos in self.r_smembers(self.rs_port_keys['port_conid_set']):
+            js = json.loads(self.r_get(pos))
+            
+            
+            print ('%s,%s -> %s,%s -> %s'%(pos, js['instr_id'], self.r_get(js['instr_id']),\
+                                           js['undly'], \
+                                           self.r_get(js['undly'])))
+        
+        
+    
+    #def cal_implvol(spot, strike, callput, evaldate, exdate, rate, div, vol, premium):
+    
+    def get_tbl_pos_csv(self, calGreeks=False):
+        s_cols = [0,1,2,3,4]
+        i_cols = [5,6,7]
+        s = '["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],'
+        
+        for l in sorted(self.port['s_pos']):
+            content = ''    
+            toks= l.split(',')
+ #           print toks
+            for i in s_cols:
+                content += '"%s",' % toks[i]
+            for i in i_cols:
+                content += '%s,' % toks[i]
+            
+                
+            s += "[%s]," % content
+        return s  
+    
+         
+
+ 
+ 
+                               
+
+    def connect(self):
+        self.con.connect()
+
+        
+    def disconnect(self):
+
+        self.con.disconnect()
+        self.quit = True
+
+if __name__ == '__main__':
+           
+    logging.basicConfig(filename = "log/port.log", filemode = 'w', 
+                        level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)-8s %(message)s')      
+    
+    config = ConfigParser.ConfigParser()
+    config.read("config/app.cfg")
+    p = PortfolioManagerEx(config)
+    p.start()
+    
+
+
+    
+#     print p.get_tbl_pos_csv()
+#     print p.get_grouped_options_str_array()
+#     print p.group_pos_by_right()
+    
+
+
+    # sample ouput    
+# ["exch","type","contract_mth","right","strike","con_ration","pos","avgcost"],["HSI","OPT","20150828","C","22600",50.0,0.0000,0.0000,],["HSI","OPT","20150828","C","23000",50.0,-1.0000,1770.0000,],["HSI","OPT","20150828","C","23600",50.0,-2.0000,1470.0000,],["HSI","OPT","20150828","C","23800",50.0,-1.0000,920.0000,],["HSI","OPT","20150828","C","24000",50.0,-2.0000,1820.0000,],["HSI","OPT","20150828","C","24200",50.0,-1.0000,3120.0000,],["HSI","OPT","20150828","C","24800",50.0,-1.0000,220.0000,],["HSI","OPT","20150828","P","18000",50.0,-2.0000,1045.0000,],["HSI","OPT","20150828","P","18600",50.0,-1.0000,1120.0000,],["HSI","OPT","20150828","P","18800",50.0,-1.0000,1570.0000,],["HSI","OPT","20150828","P","19800",50.0,-1.0000,870.0000,],["HSI","OPT","20150828","P","20200",50.0,-1.0000,970.0000,],["HSI","OPT","20150828","P","20800",50.0,-2.0000,970.0000,],["HSI","OPT","20150828","P","21600",50.0,-1.0000,1570.0000,],["HSI","OPT","20150828","P","21800",50.0,-7.0000,1955.7143,],["HSI","OPT","20150828","P","23200",50.0,1.0000,25930.0000,],["HSI","OPT","20150929","C","24400",50.0,1.0000,24880.0000,],["HSI","OPT","20150929","P","21600",50.0,0.0000,0.0000,],["HSI","OPT","20150929","P","21800",50.0,2.0000,52713.3333,],["HSI","OPT","20150929","P","22600",50.0,3.0000,39763.3333,],["MHI","OPT","20150828","C","24400",10.0,-1.0000,2603.0000,],["MHI","OPT","20150828","P","20800",10.0,-1.0000,313.0000,],["MHI","OPT","20150828","P","21000",10.0,-1.0000,363.0000,],["MHI","OPT","20150828","P","23600",10.0,5.0000,4285.0000,],["MHI","OPT","20150929","C","24400",10.0,1.0000,4947.0000,],["MHI","OPT","20150929","P","21600",10.0,1.0000,12657.0000,],["MHI","OPT","20150929","P","22600",10.0,1.0000,9877.0000,],["MHI","OPT","20150929","P","23600",10.0,4.0000,7757.0000,],
+# [180.000000,-2.0,0],[186.000000,-1.0,0],[188.000000,-1.0,0],[198.000000,-1.0,0],[202.000000,-1.0,0],[208.000000,-2.2,0],[210.000000,-0.2,0],[216.000000,-0.8,0],[218.000000,-5.0,0],[226.000000,0,0.0],[226.000000,3.2,0],[230.000000,0,-1.0],[232.000000,1.0,0],[236.000000,0,-2.0],[236.000000,1.8,0],[238.000000,0,-1.0],[240.000000,0,-2.0],[242.000000,0,-1.0],[244.000000,0,1.0],[248.000000,0,-1.0],
+# {'P': 0.0, 'C': 0.0} [('P', 0.8), ('P', 0.0), ('C', -2.0), ('C', -2.0), ('P', -1.0), ('C', 0.2), ('P', -1.0), ('P', -0.2), ('C', 0.0), ('P', -0.2), ('C', -1.0), ('P', -1.0), ('C', -1.0), ('P', 1.0), ('P', 0.2), ('P', 2.0), ('C', 1.0), ('P', -2.0), ('P', -7.0), ('P', 1.0), ('P', -1.0), ('P', -1.0), ('P', 0.2), ('C', -1.0), ('P', 3.0), ('C', -1.0), ('C', -0.2), ('P', -2.0)]
+# [('P', -8.200000000000001), ('C', -7.0)]

finopt/test1.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+import threading, logging, time
+
+from kafka.client import KafkaClient
+from kafka.consumer import SimpleConsumer
+from kafka.producer import SimpleProducer
+
+class Producer(threading.Thread):
+    daemon = True
+
+    def run(self):
+        client = KafkaClient("vsu-01:9092")
+        producer = SimpleProducer(client)#, async=True)
+
+        while True:
+            producer.send_messages('my.topic', "this is a test")
+            producer.send_messages('my.topic', "\xc2Hola, bravo!")
+
+            
+            time.sleep(1)
+
+
+class Consumer(threading.Thread):
+    daemon = True
+
+    def run(self):
+        client = KafkaClient("vsu-01:9092")
+        consumer = SimpleConsumer(client, "test-group", "my.price")
+
+        for message in consumer:
+            
+            print(message)
+
+def main():
+    threads = [
+        #Producer(),
+        Consumer()
+    ]
+
+    for t in threads:
+        t.start()
+
+    #time.sleep(5)
+    while 1:
+        pass
+    
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
+        level=logging.INFO
+        )
+    main()

finopt/test2.py

@@ -0,0 +1,271 @@
+import redis, json
+from finopt.cep.redisQueue import RedisQueue
+from numpy import *
+import pylab
+import ystockquote
+from datetime import datetime
+from scipy import stats
+
+def f1():
+    pall = set(rs.keys(pattern='PT_*'))
+    pcall = pall.difference(rs.keys(pattern='PT*P'))
+    pput = pall.difference(rs.keys(pattern='PT*C'))
+    print pall
+    print pcall
+    
+    
+    max = lambda a,b: a if (a[1] > b[1]) else b
+    min = lambda a,b: a if (a[1] < b[1]) else b
+    
+    
+    a = [(x,json.loads(rs.get(x[3:]))['5002']) for x in pall]
+    print sorted(a)
+    
+    
+    # instrument with the largest +ve delta impact
+    print reduce(max, a)
+    
+    # call instrument with the largest +ve delta impact
+    print reduce(max, [(x,json.loads(rs.get(x[3:]))['5002']) for x in pcall ])
+
+    # put instrument with the largest +ve delta impact
+    print reduce(max, [(x,json.loads(rs.get(x[3:]))['5002']) for x in pput ])
+
+    
+    # instrument with the largest -ve delta impact
+    print reduce(min,[(x,json.loads(rs.get(x[3:]))['5002']) for x in pall])
+
+
+def f2():
+    pall = set(rs.keys(pattern='PT_*'))
+    s = '["symbol","right","avgcost","spotpx","pos","delta","theta","pos_delta","pos_theta","unreal_pl","last_updated"'
+    
+    def split_toks(x):
+        pmap = json.loads(rs.get(x))
+        print pmap
+        gmap = json.loads(rs.get(x[3:]))
+        print gmap
+        s = '["%s","%s",%f,%f,%f,%f,%f,%f,%f,%f,"%s"],' % (x[3:], x[len(x)-1:], pmap['6001'], gmap['5006'], pmap['6002'],\
+                                                                     gmap['5002'],gmap['5004'],\
+                                                                     pmap['6005'],pmap['6006'],pmap['6008'],pmap['last_updated'])
+        return s                                                          
+        
+    return ''.join (split_toks( x ) for x in pall) 
+
+
+def extrapolate(ric):
+
+# data to fit
+
+    def get_hist_data():
+        l = ystockquote.get_historical_prices(ric, '20150101', '20150916')
+        #print l
+        xd = [float(x[4]) for x in l[1:]]
+        #yd = [datetime.strptime(y[0], '%Y-%m-%d') for y in l[1:]]
+        yd = [y for y in range(len(l[1:]))]
+        xd.reverse()
+        return xd, yd
+        
+
+    yd, xd = get_hist_data()
+    
+    for d in range(30, len(yd[1:]), 10):
+        x = xd[1:d]
+        y = yd[1:d]
+    #     x = random.rand(6)
+    #     y = random.rand(6)
+        
+        # fit the data with a 4th degree polynomial
+        z4 = polyfit(x, y, 4) 
+        p4 = poly1d(z4) # construct the polynomial 
+        
+        z5 = polyfit(x, y, 5)
+        p5 = poly1d(z5)
+        
+        xx = linspace(0, 350, 100)
+        pylab.plot(x, y, 'o', xx, p4(xx),'-g', xx, p5(xx),'-b', xd, yd, '-r')
+        pylab.legend(['%s to fit' % ric, '4th degree poly', '5th degree poly'])
+        #pylab.axis([0,160,0,10])
+        
+        pylab.axis([0,len(xd[1:]),min(yd) *.95,max(yd) * 1.1])
+     
+        sr, intercept, r_value, p_value, std_err = stats.linregress(range(0,5), yd[d:d+5])
+        s4, intercept, r_value, p_value, std_err = stats.linregress(range(0,5), [p4(i) for i in range(d,d+5)])
+        s5, intercept, r_value, p_value, std_err = stats.linregress(range(0,5), [p5(i) for i in range(d,d+5)])
+        print 'sr(%s):%f   s4(%s):%f    s5(%s):%f' % ('up' if sr > 0 else 'down', sr, 'up' if s4 > 0 else 'down', s4, 'up' if s5 > 0 else 'down', s5 )
+        
+        for i in range(d, d+5):
+            print '%f, %f, %f' % (yd[i], p4(i), p5(i))
+        #pylab.show()
+        pylab.savefig('./data/extrapolation/%s-%d.png' % (ric, d))
+        pylab.close()
+
+
+def extrapolate2(ric):
+
+# data to fit
+
+    def get_hist_data():
+        l = ystockquote.get_historical_prices(ric, '20150101', '20150922')
+        #print l
+        xd = [float(x[4]) for x in l[1:]]
+        #yd = [datetime.strptime(y[0], '%Y-%m-%d') for y in l[1:]]
+        yd = [y for y in range(len(l[1:]))]
+        xd.reverse()
+        return xd, yd
+        
+        
+    # x1, y1 are a subset of points of size d taken from (xr, yr) 
+    # these are points to be fed into the extrapolation function
+    # x2 provides the line space to pass to p4 and p5 -> p4(1..x2) and p5(1..x2)
+    # xr, yr contains the full set of data (each data point on y axis represents 1 day)
+    
+    def detect_trend(x1, y1, xr, yr, d):
+        z4 = polyfit(x1, y1, 4) 
+        p4 = poly1d(z4) # construct the polynomial 
+        #print y1
+        
+        z5 = polyfit(x1, y1, 5)
+        p5 = poly1d(z5)
+        
+        extrap_y_max_limit = len(x1) * 2  # 360 days
+        x2 = linspace(0, extrap_y_max_limit, 100) # 0, 160, 100 means 0 - 160 with 100 data points in between
+        pylab.plot(x1, y1, 'o', x2, p4(x2),'-g', x2, p5(x2),'-b', xr, yr, '-r')
+        pylab.legend(['%s to fit' % ric, '4th degree poly', '5th degree poly'])
+        #pylab.axis([0,160,0,10])
+        
+        pylab.axis([0,len(xr[1:])*1.1, min(yr) *.95,max(yr) * 1.1])  # first pair tells the x axis boundary, 2nd pair y axis boundary 
+        
+        # compute the slopes of each set of data points
+        # sr - slope real contains the slope computed from real data points from d to d+5 days
+        # s4 - slope extrapolated by applying 4th degree polynomial
+        sr, intercept, r_value, p_value, std_err = stats.linregress(range(0,5), yr[d:d+5])
+        s4, intercept, r_value, p_value, std_err = stats.linregress(range(0,5), [p4(i) for i in range(d,d+5)])
+        s5, intercept, r_value, p_value, std_err = stats.linregress(range(0,5), [p5(i) for i in range(d,d+5)])
+        
+        # a 3-item tuple, a value of 1 in item 1 means a correct guess, same for item 2, item 3 is just an accumulator counter used to track the number of runs  
+        rc = (1 if (sr > 0.0 and s4 > 0.0) or (sr < 0.0 and s4 < 0.0) else 0.0, 1.0 if (sr > 0.0 and s5 > 0.0) or (sr < 0.0 and s5 < 0.0) else 0.0, 1.0)
+        
+#         print 'sr(%s):%f   s4(%s):%f    s5(%s):%f' % ('up' if sr > 0 else 'down', sr, 'up' if s4 > 0 else 'down', s4, 'up' if s5 > 0 else 'down', s5 )        
+#         for i in range(d, d+5):
+#             print '%f, %f, %f' % (yr[i], p4(i), p5(i))
+        #pylab.show()
+        pylab.savefig('./data/extrapolation/%s-%d.png' % (ric, d))
+        # clear memory
+        pylab.close()
+        return rc
+
+
+
+    yd, xd = get_hist_data()
+    
+    # start with 30 points, then 40, 50, 60...
+    rcs = []
+    for d in range(30, len(yd[1:])-10, 10):
+        x = xd[1:d]
+        y = yd[1:d]
+        rcs.append(detect_trend(x, y, xd, yd, d))
+    #print rcs
+    score = reduce(lambda x, y: (x[0]+y[0], x[1]+y[1], x[2]+y[2], (x[0]+y[0]) / (x[2]+y[2]), (x[1]+y[1]) / (x[2]+y[2])), rcs)
+    print "for a total of %d iterations, p4 guessed correctly the direction of %s movement %d times (%0.4f) whereas p5 guessed correctly %d times (%0.4f)."\
+             % (score[2], ric, score[0], score[3], score[1], score[4])
+    
+    return (ric, score)
+
+
+
+        
+def mark6():
+    
+    
+    f = open('/home/larry/scribble/mark6.csv')
+    l = f.readlines()
+    n = map(lambda x: (x.split(',')), l)
+
+
+    yy=[]
+    pp=[]
+    for i in range(1,8):
+        yy.append( map(lambda x: x, [int(c[i]) for c in n]) )
+        xx = range(1, len(yy[i-1])+1)
+        print yy[i-1], xx
+        z5 = polyfit(xx, yy[i-1], 4)
+        pp.append(poly1d(z5))
+#     p1 = map(lambda x: x, [c[1] for c in n])
+#     p2 = map(lambda x: x, [c[2] for c in n])
+#     p3 = map(lambda x: x, [c[3] for c in n])
+#     p4 = map(lambda x: x, [c[4] for c in n])
+#     p5 = map(lambda x: x, [c[5] for c in n])
+#     p6 = map(lambda x: x, [c[6] for c in n])
+#     p7 = map(lambda x: x, [c[7] for c in n])   
+    print pp[0], yy[0]
+    for i in range(7):
+        print 'num: %d' % pp[i](31)       
+    xxx = linspace(0, 60, 100)
+    pylab.plot(xx, yy[0], 'o', xxx, pp[0](xxx),'-g')  
+    pylab.plot(xx, yy[1], 'o', xxx, pp[1](xxx),'-r')
+    pylab.plot(xx, yy[2], 'o', xxx, pp[2](xxx),'-b')  
+    pylab.plot(xx, yy[3], 'o', xxx, pp[3](xxx),'-c')
+    pylab.plot(xx, yy[4], 'o', xxx, pp[4](xxx),'-b')  
+    pylab.plot(xx, yy[5], 'o', xxx, pp[5](xxx),'-p')
+    
+    pylab.axis([0,50, 0,50])
+    pylab.show()
+
+
+def analyze_all(fn):
+    f = open(fn)
+    l = f.readlines()
+    rics = ['%s.HK' % s.strip('\n') for s in l]
+    return rics
+
+
+def analyze():
+    f = open ('/home/larry-13.04/workspace/finopt/data/extrapolation/stocks-extrapolation-results.txt')
+    l = f.readlines()
+    ll = eval(l[0])
+    dd = map (lambda x: (x[0], x[1][3]) if x[1][3] > 0.6 else None, ll)
+    ee = filter(lambda x: x <> None, dd)
+    print '\n'.join('%s %2f' % (x[0], x[1]) for x in ee)
+
+    
+    
+if __name__ == '__main__':
+#     rs = redis.Redis('localhost', 6379,3)
+#     
+#     s = f2()
+#     q = RedisQueue('test', host='localhost', port=6379, db=3)
+#     #[q.put(item) for item in s.split(',')]
+#     
+#     while not q.empty():
+#         q.get()
+#     [q.put(item) for item in range(1,100)]
+#     print q.qsize()
+#     #print q.qsize()
+#     print q.peek()
+#     print q.peek(50)
+    #stk = ['1398.HK', '0992.HK', '0787.HK', 'DUG', 'USO']
+    #stk = ['0700.HK', '0787.HK'] #,'0941.HK',  '2822.HK', '2823.HK', '0939.HK', '2318.HK', '1299.HK', '3988.HK', '1398.HK']
+#     rics = analyze_all('./data/hkex-stock-list.txt')
+# 
+#     st = []
+#     
+#     for s in rics[185:]:
+#         try:
+#         
+#             st.append(extrapolate2(s))
+#         except:
+#             print 'error'
+#             continue
+#             
+#     print st
+#    mark6()
+
+    analyze()
+    
+    
+    
+
+
+    
+    

finopt/trade/smart_order.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+import sys, traceback
+import json
+import logging
+import thread
+import ConfigParser
+from ib.ext.Contract import Contract
+from ib.ext.Order import Order
+from ib.opt import ibConnection, message
+from time import sleep
+import time, datetime
+import optcal
+import opt_serve
+import redis
+from helper_func.py import dict2str, str2dict
+from options_data import ContractHelper 
+import portfolio_ex
+            
+# Tick Value      Description
+# 5001            impl vol
+# 5002            delta
+# 5003            gamma
+# 5004            theta
+# 5005            vega
+# 5005            premium
+# IB tick types code reference
+# https://www.interactivebrokers.com/en/software/api/api.htm
+                    
+
+class SmartOrderSelector(object):
+    
+    ORDER_SIDE_BUYBACK = 10000
+    ORDER_SIDE_SELLOFF = 10100
+    
+    def __init__(self):
+        pass
+        
+    def refresh_latest_pos(self):
+        print 'refreshLatestPosition'
+    
+
+    
+    def rule_IV(self, mode):
+        
+        pass
+    
+    def rule_spread(self):
+        pass
+    
+    def rule_theta(self):
+        pass
+    
+    def rule_month(self):
+        pass
+    
+    def rule_dist_from_spot(self):
+        pass
+    
+    def all_call_pos(self):
+        pass
+    
+    def all_put_pos(self):
+        pass
+    
+    
+    def select_best_order(self):
+        # return the best order from the portfolio to trade
+        return Order()
+    
+    def __str__(self):
+        return ''.join(('%s=%s')%(k,v) for k, v in self.__dict__.iteritems())
+
+class UnwindOrderSelector(SmartOrderSelector):
+    right_type = None
+    
+    def __init__(self, right_type. rule):
+        self.right_type = right_type
+        
+    
+    def pos_min_IV(self):
+        Contracts = []
+        
+        return []
+        
+    def rule_IV(self, mode):
+        print 'unwindorderselector:rule_iv'
+        if mode == self.ORDER_SIDE_BUYBACK:
+            # returns a list of the top three instrument with the 
+            # lowest IV in ascending order
+            # return self.pos_min_IV()
+            
+            pass
+        elif mode == self.ORDER_SIDE_SELLOFF:
+            # return self.pos_max_IV()
+            pass
+
+    
+    def rule_spread(self, mode):
+        print 'unwindorderselector:rule_iv'
+        if mode == self.ORDER_SIDE_BUYBACK:
+            # returns a list of the top three instrument with the 
+            # lowest IV in ascending order
+            # return self.pos_min_IV()
+            
+            pass
+        elif mode == self.ORDER_SIDE_SELLOFF:
+            # return self.pos_max_IV()
+            pass
+        
+        
+     
+        
+
+if __name__ == '__main__':
+           
+    logging.basicConfig(#filename = "log/opt.log", filemode = 'w', 
+                        level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)-8s %(message)s')      
+    
+    config = ConfigParser.ConfigParser()
+    config.read("config/app.cfg")
+    
+    x = SmartOrderSelector()
+    print x
+    y = UnwindOrderSelector('c')
+    y.rule_IV()
+    print y

finopt/ystockquote.py

@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2007-2008, Corey Goldberg (corey@goldb.org)
+#
+#  license: GNU LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+
+
+import urllib
+
+
+"""
+This is the "ystockquote" module.
+
+This module provides a Python API for retrieving stock data from Yahoo Finance.
+
+sample usage:
+>>> import ystockquote
+>>> print ystockquote.get_price('GOOG')
+529.46
+
+
+
+
+http://www.gummy-stuff.org/Yahoo-data.htm
+a     Ask     a2     Average Daily Volume     a5     Ask Size
+b     Bid     b2     Ask (Real-time)     b3     Bid (Real-time)
+b4     Book Value     b6     Bid Size     c     Change & Percent Change
+c1     Change     c3     Commission     c6     Change (Real-time)
+c8     After Hours Change (Real-time)     d     Dividend/Share     d1     Last Trade Date
+d2     Trade Date     e     Earnings/Share     e1     Error Indication (returned for symbol changed / invalid)
+e7     EPS Estimate Current Year     e8     EPS Estimate Next Year     e9     EPS Estimate Next Quarter
+f6     Float Shares     g     Day's Low     h     Day's High
+j     52-week Low     k     52-week High     g1     Holdings Gain Percent
+g3     Annualized Gain     g4     Holdings Gain     g5     Holdings Gain Percent (Real-time)
+g6     Holdings Gain (Real-time)     i     More Info     i5     Order Book (Real-time)
+j1     Market Capitalization     j3     Market Cap (Real-time)     j4     EBITDA
+j5     Change From 52-week Low     j6     Percent Change From 52-week Low     k1     Last Trade (Real-time) With Time
+k2     Change Percent (Real-time)     k3     Last Trade Size     k4     Change From 52-week High
+k5     Percebt Change From 52-week High     l     Last Trade (With Time)     l1     Last Trade (Price Only)
+l2     High Limit     l3     Low Limit     m     Day's Range
+m2     Day's Range (Real-time)     m3     50-day Moving Average     m4     200-day Moving Average
+m5     Change From 200-day Moving Average     m6     Percent Change From 200-day Moving Average     m7     Change From 50-day Moving Average
+m8     Percent Change From 50-day Moving Average     n     Name     n4     Notes
+o     Open     p     Previous Close     p1     Price Paid
+p2     Change in Percent     p5     Price/Sales     p6     Price/Book
+q     Ex-Dividend Date     r     P/E Ratio     r1     Dividend Pay Date
+r2     P/E Ratio (Real-time)     r5     PEG Ratio     r6     Price/EPS Estimate Current Year
+r7     Price/EPS Estimate Next Year     s     Symbol     s1     Shares Owned
+s7     Short Ratio     t1     Last Trade Time     t6     Trade Links
+t7     Ticker Trend     t8     1 yr Target Price     v     Volume
+v1     Holdings Value     v7     Holdings Value (Real-time)     w     52-week Range
+w1     Day's Value Change     w4     Day's Value Change (Real-time)     x     Stock Exchange
+y     Dividend Yield         
+
+"""
+
+
+def __request(symbol, stat):
+    url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat)
+    return urllib.urlopen(url).read().strip().strip('"')
+
+
+def get_all(symbol):
+    """
+    Get all available quote data for the given ticker symbol.
+    
+    Returns a dictionary.
+    """
+    values = __request(symbol, 'l1c1va2xj1b4j4dyekjm3m4rr5p5p6s7').split(',')
+    data = {}
+    data['price'] = values[0]
+    data['change'] = values[1]
+    data['volume'] = values[2]
+    data['avg_daily_volume'] = values[3]
+    data['stock_exchange'] = values[4]
+    data['market_cap'] = values[5]
+    data['book_value'] = values[6]
+    data['ebitda'] = values[7]
+    data['dividend_per_share'] = values[8]
+    data['dividend_yield'] = values[9]
+    data['earnings_per_share'] = values[10]
+    data['52_week_high'] = values[11]
+    data['52_week_low'] = values[12]
+    data['50day_moving_avg'] = values[13]
+    data['200day_moving_avg'] = values[14]
+    data['price_earnings_ratio'] = values[15]
+    data['price_earnings_growth_ratio'] = values[16]
+    data['price_sales_ratio'] = values[17]
+    data['price_book_ratio'] = values[18]
+    data['short_ratio'] = values[19]
+    return data
+    
+    
+def get_price(symbol): 
+    return __request(symbol, 'l1')
+
+def get_close(symbol):
+    return __request(symbol, 'p')
+
+def get_change(symbol):
+    return __request(symbol, 'c1')
+    
+    
+def get_volume(symbol): 
+    return __request(symbol, 'v')
+
+
+def get_avg_daily_volume(symbol): 
+    return __request(symbol, 'a2')
+    
+    
+def get_stock_exchange(symbol): 
+    return __request(symbol, 'x')
+    
+    
+def get_market_cap(symbol):
+    return __request(symbol, 'j1')
+   
+   
+def get_book_value(symbol):
+    return __request(symbol, 'b4')
+
+
+def get_ebitda(symbol): 
+    return __request(symbol, 'j4')
+    
+    
+def get_dividend_per_share(symbol):
+    return __request(symbol, 'd')
+
+
+def get_dividend_yield(symbol): 
+    return __request(symbol, 'y')
+    
+    
+def get_earnings_per_share(symbol): 
+    return __request(symbol, 'e')
+
+
+def get_52_week_high(symbol): 
+    return __request(symbol, 'k')
+    
+    
+def get_52_week_low(symbol): 
+    return __request(symbol, 'j')
+
+
+def get_50day_moving_avg(symbol): 
+    return __request(symbol, 'm3')
+    
+    
+def get_200day_moving_avg(symbol): 
+    return __request(symbol, 'm4')
+    
+    
+def get_price_earnings_ratio(symbol): 
+    return __request(symbol, 'r')
+
+
+def get_price_earnings_growth_ratio(symbol): 
+    return __request(symbol, 'r5')
+
+
+def get_price_sales_ratio(symbol): 
+    return __request(symbol, 'p5')
+    
+    
+def get_price_book_ratio(symbol): 
+    return __request(symbol, 'p6')
+       
+       
+def get_short_ratio(symbol): 
+    return __request(symbol, 's7')
+    
+    
+def get_historical_prices(symbol, start_date, end_date):
+    """
+    Get historical prices for the given ticker symbol.
+    Date format is 'YYYYMMDD'
+    
+    Returns a nested list.
+    """
+    url = 'http://ichart.yahoo.com/table.csv?s=%s&' % symbol + \
+          'd=%s&' % str(int(end_date[4:6]) - 1) + \
+          'e=%s&' % str(int(end_date[6:8])) + \
+          'f=%s&' % str(int(end_date[0:4])) + \
+          'g=d&' + \
+          'a=%s&' % str(int(start_date[4:6]) - 1) + \
+          'b=%s&' % str(int(start_date[6:8])) + \
+          'c=%s&' % str(int(start_date[0:4])) + \
+          'ignore=.csv'
+    days = urllib.urlopen(url).readlines()
+    data = [day[:-2].split(',') for day in days]
+    return data
+
+
+#ss = [["0001.HK",2.5],["0002.HK",1.86],["0003.HK",1.62],["0004.HK",1.27],["0005.HK",15.02],["0006.HK",1.44],["0011.HK",1.42],["0012.HK",0.82],["0013.HK",2.53],["0016.HK",2.53],["0017.HK",0.7],["0019.HK",1.01],["0023.HK",0.67],["0066.HK",0.67],["0083.HK",0.66],["0101.HK",0.94],["0144.HK",0.46],["0151.HK",1.14],["0267.HK",0.18],["0291.HK",0.46],["0293.HK",0.24],["0322.HK",0.71],["0330.HK",0.25],["0386.HK",1.88],["0388.HK",1.86],["0494.HK",1.06],["0688.HK",1.25],["0700.HK",4.52],["0762.HK",0.94],["0836.HK",0.49],["0857.HK",3.29],["0883.HK",4.38],["0939.HK",7.08],["0941.HK",8.08],["1044.HK",0.91],["1088.HK",1.59],["1109.HK",0.53],["1199.HK",0.27],["1299.HK",4.69],["1398.HK",5.25],["1880.HK",1.01],["1898.HK",0.43],["1928.HK",1.06],["2318.HK",1.85],["2388.HK",1.42],["2600.HK",0.19],["2628.HK",2.56],["3328.HK",0.71],["3988.HK",3.63]]
+#for s in ss:
+#    chg_percent = float(get_change(s[0]))/ float(get_price(s[0])) 
+#    print '["%s","HSI",%f,%f],' % (s[0], s[1], chg_percent)
+    
+    
+