minor updates

cep/t1.py

@@ -9,7 +9,8 @@ import numpy
 import time, datetime
 from os import listdir
 from os.path import isfile, join
-
+from optparse import OptionParser
+import sys
 path = '/home/larry/l1304/workspace/finopt/data/mds_files/large_up_1002/ibkdump-20151002105314.txt'
 path = '/home/larry/l1304/workspace/finopt/data/mds_files/large_up_1002/ibkdump-20151002110412.txt'
 
@@ -78,9 +79,25 @@ def process_msg_by_key(file):
     
 
 if __name__ == '__main__':
+    
+    parser = OptionParser()
+#     parser.add_option("-r", "--replay",
+#                       dest="replay_dir",
+#                       help="replay recorded mds files stored in the specified directory")
+                      
+    
+    options, arguments = parser.parse_args()
+
+    #print options, arguments
+    
+    if len(sys.argv) < 2:
+        print("Usage: %s [options] <dir>" % sys.argv[0])
+        exit(-1)    
+    
+    
     sc = SparkContext(appName= 't1')    
     #dir_loc = '/home/larry/l1304/workspace/finopt/data/mds_files/large_up_1002'
-    dir_loc = '/home/larry/l1304/workspace/finopt/data/mds_files'
+    dir_loc = arguments[0]
     files = sorted([ join(dir_loc,f) for f in listdir(dir_loc) if isfile(join(dir_loc,f)) ])
 
     a = [(process_msg_by_key(f)) for f in files]

config/mds.cfg

@@ -1,44 +1,9 @@
-[global]
-server.socket_host: "0.0.0.0"
-server.socket_port: 8082
-
 [redis]
 redis.server: "localhost"
 redis.port: 6379
 redis.db: 3
 redis.sleep: 0.5
-redis.datastore.key.option_implv: 'opt_implv'
-redis.datastore.key.option_chains: 'opt_chains'
-redis.datastore.key.option_set: 'opt_set'
-redis.datastore.key.option_implv_ts_set: 'opt_implv_ts_set'
-redis.datastore.key.option_implv_ts: 'opt_implv_ts' 
-
-# a redis key that stores all the contract ids of contracts in a portfolio  
-redis.datastore.key.port_conid_set: 'port_conid_set' 
-redis.datastore.key.port_prefix: 'PT'
-# note that the port summary key has no prefix assigned to it
-# to retrive its value just use rs.get instead of self.rs_<xxx>
-redis.datastore.key.port_summary: 'port_summary'
- 
-[/]
-tools.sessions.on : True
-tools.staticdir.root : '/home/larry-13.04/workspace/finopt/src/'
-#tools.staticdir.root : '/home/larry-13.04/production/finopt/'
-
-[/static]
-tools.staticdir.on : True
-#tools.staticdir.dir : './public'
-tools.staticdir.tmpl : './html'
-
-
-[options_data]
-options_data.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/opt.log', 'filemode': 'w','level': logging.DEBUG}"
 
-[portfolio]
-portfolio.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/port.log', 'filemode': 'w','level': logging.INFO}"
-
-[opt_serve]
-opt_serve.logconfig: "{'filename': '/home/larry-13.04/workspace/finopt/log/opt.log', 'filemode': 'w','level': logging.INFO}"
 
 [cep]
 kafka.host: 'vsu-01'
@@ -47,24 +12,6 @@ kafka.ib.topic.tick_price: 'ib_tick_price'
 kafka.ib.topic.tick_size: 'ib_tick_size'
 
 
-
-[market]
-hkex.openhours: '{"morning":[915,1200], "afternoon":[1300,1615]}'
-ib.gateway: '127.0.0.1'
-ib.port: 8496
-#gw port
-#ib.port:4001
-ib.appid.portfolio: 9922
-
-ib.appid: 9911
-option.underlying = "('HSI', 'FUT', 'HKFE', 'HKD', '', 0, '')"
-option.underlying.month_price = "[['20150929', 22817.0, '^HSI'], ['20151029', 22715.0, '^HSI']]"
-option.underlying.yahoo_ws = "{'use_yahoo': True, 'func': 'ystockquote.get_price'}"
-option.underlying.tick_size = 200
-option.greeks.recal = "{'use_last_if_no_bidask': True, 'rate':0.005, 'div':0.005, 'vol':0.2}"
-option.chain_range = 0.08
-option.bid_ask_spread_tolerance = 0.90
-
 [alert_bot]
 msg_bot.jid: "robo@route69.hopto.org"
 msg_bot.pass: 123

finopt/test1.py

@@ -33,7 +33,7 @@ class Consumer(threading.Thread):
 
 def main():
     threads = [
-        #Producer(),
+        Producer(),
         Consumer()
     ]
 

sh/t1.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
-ROOT=/home/larry-13.04/workspace/finopt
-SRC=$ROOT/src
+
+ROOT=$FINOPT_HOME
+FINDATA=$ROOT/../data 
+SRC=$ROOT
 KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
 export PYTHONPATH=$SRC:$PYTHONPATH
 
 #spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
 #spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
 #spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
-spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py 
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py $FINDATA/mds_files/large_up_1002