all day coding

add a logger filter to reduce too many output
add a new table to show port summary in front end
add a pointer to show HSI futures value
add a new function to derive HSIF last trading day
fix greek calculation errors - add sanity checks
use close price if last px is not avail
filter zero positions to avoid computation issues
include closing price in tick data in tds_event
ws add routines to listen tws market data

src/config/daily_download.cfg

@@ -0,0 +1,66 @@
+[run_config]
+logconfig: {'level': logging.INFO, 'filemode': 'w', 'filename': '/tmp/daily_download.log'}
+output_path:'/home/laxaurus/workspace/finopt/data/'
+data_list: "{'a':['HSI options after market HTML', 'dha_url'],'b':['HSI options after market ZIP', 'dza_url'],'c':['HSI options normal hours ZIP', 'dzn_url'],'d':['HSI futures normal hours HTML', 'fhn_url'],'e':['HSI futures normal hours ZIP', 'fzn_url'],'f':['HHI CN futures normal hours HTML', 'hhn_url'],'g':['HHI CN futures normal hours ZIP', 'hzn_url'],'h':['HHI CN options normal hours HTML', 'ohn_url'],'i':['HHI CN options normal hours ZIP', 'ozn_url'],'j':['Cash market daily quotes HTML ', 'shd_url'],'k':['Cash market short sell HTML', 'shs_url'],'l':['HSI volatility HTML', 'vh_url'],'m':['HSI volatility ZIP', 'vz_url']}"
+
+
+[holiday]
+hk_holiday: "{'2019': ['0101','0205','0206','0207','0405','0419','0420','0422','0501','0513','0607','0701','0914','1001','1007','1225','1226']}"
+
+[hkex_data]
+#
+# Derivative market
+#
+# d - derivative options
+# z - zip / h - html
+# n - normal trading / a - after hours
+
+	 
+# Day Trading Session zip		
+# %s - YYMMDD
+# 
+#
+dzn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hsio%s.zip'
+dhn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hsio%s.htm'
+
+
+# After-Hours Trading Session		
+dza_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hsioa%s.zip'
+dha_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hsioa%s.htm'
+
+
+#
+# Futures
+#
+# f - futures HSI
+fzn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hsif%s.zip'
+fhn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hsif%s.htm'
+# 
+# h - china enterprise
+# 
+hzn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hhif%s.zip'
+hhn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hhif%s.htm'
+#
+# o - china enterprise options
+# 
+ozn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hhio%s.zip'
+ohn_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/hhio%s.htm'
+#
+# v - hsi volatility
+#
+vz_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/vhsf%s.zip'
+vh_url: 'https://www.hkex.com.hk/eng/stat/dmstat/dayrpt/vhsf%s.htm'
+
+#
+# Securities market
+#
+# s - securities
+# h - htm
+# d - daily quote | s - short sell
+#
+# https://www.hkex.com.hk/eng/stat/smstat/dayquot/d190205e.htm
+shd_url: 'https://www.hkex.com.hk/eng/stat/smstat/dayquot/d%se.htm'
+
+# short selling 
+# main board
+shs_url: 'https://www.hkex.com.hk/Market-Data/Statistics/Securities-Market/Short-Selling-Turnover-Today/Short-Selling-Turnover-(Main-Board)-up-to-morning-close-today?sc_lang=en'

src/hkex/daily_download.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
-import sys, traceback
+import traceback
 import logging
 from misc2.helpers import ConfigMap
 from optparse import OptionParser
 import time, datetime
 import os, wget
-import json, ntpath
+import ntpath
 
 
 class HkexStatDownloader():
@@ -63,8 +63,11 @@ class HkexStatDownloader():
             
             
             # special check for 'k' type dayily short sell
+            # if the date supplied is not today's date, the download is not available 
             if day_str <> datetime.datetime.now().strftime('%y%m%d') and ch == 'k':
                 continue
+            # there is no need to concat day str for daily download html page
+            # for other downloads, insert the day str into the url
             link = link % day_str if ch <> 'k' else link
             try:
                 path = '%s/%s' % (dir, ntpath.basename(link))