cep changes

cep/t1.py

@@ -25,7 +25,7 @@ path = '/home/larry/l1304/workspace/finopt/data/mds_files/large_up_1002/ibkdump-
 def process_msg(file):
 
     try:
-        md = sc.textFile(file)    
+        md = sc.textFile(file)
         mdl = md.map(lambda lines: (lines.split('|'))).map(lambda x: json.loads(x[1]))\
                 .filter(lambda x: (x['typeName'] == 'tickPrice' and x['contract'] == "HSI-20151029-0--FUT-HKD-102"))\
                 .map(lambda x: x['price'])

sh/t1.sh

@@ -7,5 +7,5 @@ export PYTHONPATH=$SRC:$PYTHONPATH
 #spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
 #spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
 #spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
-spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py 
 

sh/t1.sh~

@@ -0,0 +1,11 @@
+#!/bin/bash
+ROOT=/home/larry-13.04/workspace/finopt
+SRC=$ROOT/src
+KAFKA_ASSEMBLY_JAR=$ROOT/src/jar/spark-streaming-kafka-assembly_2.10-1.4.1.jar
+export PYTHONPATH=$SRC:$PYTHONPATH
+
+#spark-submit  --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+#spark-submit --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 simple 
+#spark-submit --total-executor-cores 2 --master spark://192.168.1.118:7077   --jars  $KAFKA_ASSEMBLY_JAR /home/larry-13.04/workspace/finopt/cep/momentum.py vsu-01:2181 hsi 1 cal_trend 
+spark-submit  --jars  $KAFKA_ASSEMBLY_JAR $SRC/cep/t1.py
+