{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86d10ac2",
   "metadata": {},
   "source": [
    "## first test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0ef32585",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Historical Data:\n",
      "                                 Open        High         Low       Close  \\\n",
      "Date                                                                        \n",
      "2024-08-12 00:00:00-04:00  215.069080  218.493132  214.601256  216.522308   \n",
      "2024-08-13 00:00:00-04:00  217.995439  220.862103  217.995439  220.244980   \n",
      "2024-08-14 00:00:00-04:00  219.548220  221.996815  218.682240  220.692886   \n",
      "2024-08-15 00:00:00-04:00  223.559546  224.306072  221.728059  223.678986   \n",
      "2024-08-16 00:00:00-04:00  222.882700  225.779224  222.613947  225.002838   \n",
      "...                               ...         ...         ...         ...   \n",
      "2025-08-05 00:00:00-04:00  203.169408  205.107211  201.930824  202.689957   \n",
      "2025-08-06 00:00:00-04:00  205.396898  215.135845  205.356935  213.008255   \n",
      "2025-08-07 00:00:00-04:00  218.631873  220.599641  216.334477  219.780563   \n",
      "2025-08-08 00:00:00-04:00  220.579666  230.738135  219.001455  229.090012   \n",
      "2025-08-11 00:00:00-04:00  227.919998  229.559998  224.759995  227.179993   \n",
      "\n",
      "                              Volume  Dividends  Stock Splits  \n",
      "Date                                                           \n",
      "2024-08-12 00:00:00-04:00   38028100       0.25           0.0  \n",
      "2024-08-13 00:00:00-04:00   44155300       0.00           0.0  \n",
      "2024-08-14 00:00:00-04:00   41960600       0.00           0.0  \n",
      "2024-08-15 00:00:00-04:00   46414000       0.00           0.0  \n",
      "2024-08-16 00:00:00-04:00   44340200       0.00           0.0  \n",
      "...                              ...        ...           ...  \n",
      "2025-08-05 00:00:00-04:00   44155100       0.00           0.0  \n",
      "2025-08-06 00:00:00-04:00  108483100       0.00           0.0  \n",
      "2025-08-07 00:00:00-04:00   90224800       0.00           0.0  \n",
      "2025-08-08 00:00:00-04:00  113854000       0.00           0.0  \n",
      "2025-08-11 00:00:00-04:00   61725300       0.26           0.0  \n",
      "\n",
      "[250 rows x 7 columns]\n",
      "\n",
      "Financials:\n",
      "                                                      2024-09-30  \\\n",
      "Tax Effect Of Unusual Items                         0.000000e+00   \n",
      "Tax Rate For Calcs                                  2.410000e-01   \n",
      "Normalized EBITDA                                   1.346610e+11   \n",
      "Net Income From Continuing Operation Net Minori...  9.373600e+10   \n",
      "Reconciled Depreciation                             1.144500e+10   \n",
      "Reconciled Cost Of Revenue                          2.103520e+11   \n",
      "EBITDA                                              1.346610e+11   \n",
      "EBIT                                                1.232160e+11   \n",
      "Net Interest Income                                          NaN   \n",
      "Interest Expense                                             NaN   \n",
      "Interest Income                                              NaN   \n",
      "Normalized Income                                   9.373600e+10   \n",
      "Net Income From Continuing And Discontinued Ope...  9.373600e+10   \n",
      "Total Expenses                                      2.678190e+11   \n",
      "Total Operating Income As Reported                  1.232160e+11   \n",
      "Diluted Average Shares                              1.540810e+10   \n",
      "Basic Average Shares                                1.534378e+10   \n",
      "Diluted EPS                                         6.080000e+00   \n",
      "Basic EPS                                           6.110000e+00   \n",
      "Diluted NI Availto Com Stockholders                 9.373600e+10   \n",
      "Net Income Common Stockholders                      9.373600e+10   \n",
      "Net Income                                          9.373600e+10   \n",
      "Net Income Including Noncontrolling Interests       9.373600e+10   \n",
      "Net Income Continuous Operations                    9.373600e+10   \n",
      "Tax Provision                                       2.974900e+10   \n",
      "Pretax Income                                       1.234850e+11   \n",
      "Other Income Expense                                2.690000e+08   \n",
      "Other Non Operating Income Expenses                 2.690000e+08   \n",
      "Net Non Operating Interest Income Expense                    NaN   \n",
      "Interest Expense Non Operating                               NaN   \n",
      "Interest Income Non Operating                                NaN   \n",
      "Operating Income                                    1.232160e+11   \n",
      "Operating Expense                                   5.746700e+10   \n",
      "Research And Development                            3.137000e+10   \n",
      "Selling General And Administration                  2.609700e+10   \n",
      "Gross Profit                                        1.806830e+11   \n",
      "Cost Of Revenue                                     2.103520e+11   \n",
      "Total Revenue                                       3.910350e+11   \n",
      "Operating Revenue                                   3.910350e+11   \n",
      "\n",
      "                                                      2023-09-30  \\\n",
      "Tax Effect Of Unusual Items                         0.000000e+00   \n",
      "Tax Rate For Calcs                                  1.470000e-01   \n",
      "Normalized EBITDA                                   1.258200e+11   \n",
      "Net Income From Continuing Operation Net Minori...  9.699500e+10   \n",
      "Reconciled Depreciation                             1.151900e+10   \n",
      "Reconciled Cost Of Revenue                          2.141370e+11   \n",
      "EBITDA                                              1.258200e+11   \n",
      "EBIT                                                1.143010e+11   \n",
      "Net Interest Income                                -1.830000e+08   \n",
      "Interest Expense                                    3.933000e+09   \n",
      "Interest Income                                     3.750000e+09   \n",
      "Normalized Income                                   9.699500e+10   \n",
      "Net Income From Continuing And Discontinued Ope...  9.699500e+10   \n",
      "Total Expenses                                      2.689840e+11   \n",
      "Total Operating Income As Reported                  1.143010e+11   \n",
      "Diluted Average Shares                              1.581255e+10   \n",
      "Basic Average Shares                                1.574423e+10   \n",
      "Diluted EPS                                         6.130000e+00   \n",
      "Basic EPS                                           6.160000e+00   \n",
      "Diluted NI Availto Com Stockholders                 9.699500e+10   \n",
      "Net Income Common Stockholders                      9.699500e+10   \n",
      "Net Income                                          9.699500e+10   \n",
      "Net Income Including Noncontrolling Interests       9.699500e+10   \n",
      "Net Income Continuous Operations                    9.699500e+10   \n",
      "Tax Provision                                       1.674100e+10   \n",
      "Pretax Income                                       1.137360e+11   \n",
      "Other Income Expense                               -5.650000e+08   \n",
      "Other Non Operating Income Expenses                -5.650000e+08   \n",
      "Net Non Operating Interest Income Expense          -1.830000e+08   \n",
      "Interest Expense Non Operating                      3.933000e+09   \n",
      "Interest Income Non Operating                       3.750000e+09   \n",
      "Operating Income                                    1.143010e+11   \n",
      "Operating Expense                                   5.484700e+10   \n",
      "Research And Development                            2.991500e+10   \n",
      "Selling General And Administration                  2.493200e+10   \n",
      "Gross Profit                                        1.691480e+11   \n",
      "Cost Of Revenue                                     2.141370e+11   \n",
      "Total Revenue                                       3.832850e+11   \n",
      "Operating Revenue                                   3.832850e+11   \n",
      "\n",
      "                                                      2022-09-30  \\\n",
      "Tax Effect Of Unusual Items                         0.000000e+00   \n",
      "Tax Rate For Calcs                                  1.620000e-01   \n",
      "Normalized EBITDA                                   1.305410e+11   \n",
      "Net Income From Continuing Operation Net Minori...  9.980300e+10   \n",
      "Reconciled Depreciation                             1.110400e+10   \n",
      "Reconciled Cost Of Revenue                          2.235460e+11   \n",
      "EBITDA                                              1.305410e+11   \n",
      "EBIT                                                1.194370e+11   \n",
      "Net Interest Income                                -1.060000e+08   \n",
      "Interest Expense                                    2.931000e+09   \n",
      "Interest Income                                     2.825000e+09   \n",
      "Normalized Income                                   9.980300e+10   \n",
      "Net Income From Continuing And Discontinued Ope...  9.980300e+10   \n",
      "Total Expenses                                      2.748910e+11   \n",
      "Total Operating Income As Reported                  1.194370e+11   \n",
      "Diluted Average Shares                              1.632582e+10   \n",
      "Basic Average Shares                                1.621596e+10   \n",
      "Diluted EPS                                         6.110000e+00   \n",
      "Basic EPS                                           6.150000e+00   \n",
      "Diluted NI Availto Com Stockholders                 9.980300e+10   \n",
      "Net Income Common Stockholders                      9.980300e+10   \n",
      "Net Income                                          9.980300e+10   \n",
      "Net Income Including Noncontrolling Interests       9.980300e+10   \n",
      "Net Income Continuous Operations                    9.980300e+10   \n",
      "Tax Provision                                       1.930000e+10   \n",
      "Pretax Income                                       1.191030e+11   \n",
      "Other Income Expense                               -3.340000e+08   \n",
      "Other Non Operating Income Expenses                -3.340000e+08   \n",
      "Net Non Operating Interest Income Expense          -1.060000e+08   \n",
      "Interest Expense Non Operating                      2.931000e+09   \n",
      "Interest Income Non Operating                       2.825000e+09   \n",
      "Operating Income                                    1.194370e+11   \n",
      "Operating Expense                                   5.134500e+10   \n",
      "Research And Development                            2.625100e+10   \n",
      "Selling General And Administration                  2.509400e+10   \n",
      "Gross Profit                                        1.707820e+11   \n",
      "Cost Of Revenue                                     2.235460e+11   \n",
      "Total Revenue                                       3.943280e+11   \n",
      "Operating Revenue                                   3.943280e+11   \n",
      "\n",
      "                                                      2021-09-30    2020-09-30  \n",
      "Tax Effect Of Unusual Items                         0.000000e+00           NaN  \n",
      "Tax Rate For Calcs                                  1.330000e-01           NaN  \n",
      "Normalized EBITDA                                   1.231360e+11           NaN  \n",
      "Net Income From Continuing Operation Net Minori...  9.468000e+10           NaN  \n",
      "Reconciled Depreciation                             1.128400e+10           NaN  \n",
      "Reconciled Cost Of Revenue                          2.129810e+11           NaN  \n",
      "EBITDA                                              1.231360e+11           NaN  \n",
      "EBIT                                                1.118520e+11           NaN  \n",
      "Net Interest Income                                 1.980000e+08  8.900000e+08  \n",
      "Interest Expense                                    2.645000e+09  2.873000e+09  \n",
      "Interest Income                                     2.843000e+09  3.763000e+09  \n",
      "Normalized Income                                   9.468000e+10           NaN  \n",
      "Net Income From Continuing And Discontinued Ope...  9.468000e+10           NaN  \n",
      "Total Expenses                                      2.568680e+11           NaN  \n",
      "Total Operating Income As Reported                  1.089490e+11           NaN  \n",
      "Diluted Average Shares                              1.686492e+10           NaN  \n",
      "Basic Average Shares                                1.670127e+10           NaN  \n",
      "Diluted EPS                                         5.610000e+00           NaN  \n",
      "Basic EPS                                           5.670000e+00           NaN  \n",
      "Diluted NI Availto Com Stockholders                 9.468000e+10           NaN  \n",
      "Net Income Common Stockholders                      9.468000e+10           NaN  \n",
      "Net Income                                          9.468000e+10           NaN  \n",
      "Net Income Including Noncontrolling Interests       9.468000e+10           NaN  \n",
      "Net Income Continuous Operations                    9.468000e+10           NaN  \n",
      "Tax Provision                                       1.452700e+10           NaN  \n",
      "Pretax Income                                       1.092070e+11           NaN  \n",
      "Other Income Expense                                6.000000e+07           NaN  \n",
      "Other Non Operating Income Expenses                 6.000000e+07           NaN  \n",
      "Net Non Operating Interest Income Expense           1.980000e+08  8.900000e+08  \n",
      "Interest Expense Non Operating                      2.645000e+09  2.873000e+09  \n",
      "Interest Income Non Operating                       2.843000e+09  3.763000e+09  \n",
      "Operating Income                                    1.089490e+11           NaN  \n",
      "Operating Expense                                   4.388700e+10           NaN  \n",
      "Research And Development                            2.191400e+10           NaN  \n",
      "Selling General And Administration                  2.197300e+10           NaN  \n",
      "Gross Profit                                        1.528360e+11           NaN  \n",
      "Cost Of Revenue                                     2.129810e+11           NaN  \n",
      "Total Revenue                                       3.658170e+11           NaN  \n",
      "Operating Revenue                                   3.658170e+11           NaN  \n",
      "\n",
      "Stock Actions:\n",
      "                           Dividends  Stock Splits\n",
      "Date                                              \n",
      "1987-05-11 00:00:00-04:00   0.000536           0.0\n",
      "1987-06-16 00:00:00-04:00   0.000000           2.0\n",
      "1987-08-10 00:00:00-04:00   0.000536           0.0\n",
      "1987-11-17 00:00:00-05:00   0.000714           0.0\n",
      "1988-02-12 00:00:00-05:00   0.000714           0.0\n",
      "...                              ...           ...\n",
      "2024-08-12 00:00:00-04:00   0.250000           0.0\n",
      "2024-11-08 00:00:00-05:00   0.250000           0.0\n",
      "2025-02-10 00:00:00-05:00   0.250000           0.0\n",
      "2025-05-12 00:00:00-04:00   0.260000           0.0\n",
      "2025-08-11 00:00:00-04:00   0.260000           0.0\n",
      "\n",
      "[93 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "import yfinance as yf\n",
    "\n",
    "# Define the ticker symbol\n",
    "ticker_symbol = \"AAPL\"\n",
    "\n",
    "# Create a Ticker object\n",
    "ticker = yf.Ticker(ticker_symbol)\n",
    "\n",
    "# Fetch historical market data\n",
    "historical_data = ticker.history(period=\"1y\")  # data for the last year\n",
    "print(\"Historical Data:\")\n",
    "print(historical_data)\n",
    "\n",
    "# Fetch basic financials\n",
    "financials = ticker.financials\n",
    "print(\"\\nFinancials:\")\n",
    "print(financials)\n",
    "\n",
    "# Fetch stock actions like dividends and splits\n",
    "actions = ticker.actions\n",
    "print(\"\\nStock Actions:\")\n",
    "print(actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "47d70a61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2528194/3537653429.py:15: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  ydat = yf.download(tickers, start='2019-01-01', end='2024-06-01')\n",
      "[*********************100%***********************]  10 of 10 completed\n",
      "/tmp/ipykernel_2528194/3537653429.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data.dropna(inplace=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
       "columns": [
        {
         "name": "Date",
         "rawType": "datetime64[ns]",
         "type": "datetime"
        },
        {
         "name": "ADA-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "AVAX-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "BNB-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "BTC-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "DOGE-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "DOT-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "ETH-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "SOL-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "TRX-USD",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "XRP-USD",
         "rawType": "float64",
         "type": "float"
        }
       ],
       "ref": "5c246526-c932-45aa-be3e-6a4e6ff0a196",
       "rows": [
        [
         "2020-09-22 00:00:00",
         "0.08175399899482727",
         "5.2346320152282715",
         "24.100740432739258",
         "10538.4599609375",
         "0.002655999967828393",
         "4.174462795257568",
         "344.503173828125",
         "2.9155189990997314",
         "0.025043999776244164",
         "0.23341700434684753"
        ],
        [
         "2020-09-23 00:00:00",
         "0.07676299661397934",
         "4.11846923828125",
         "22.856977462768555",
         "10246.1865234375",
         "0.0025599999353289604",
         "4.031414985656738",
         "321.1163024902344",
         "2.836756944656372",
         "0.024858999997377396",
         "0.22122099995613098"
        ],
        [
         "2020-09-24 00:00:00",
         "0.08281700313091278",
         "4.566561222076416",
         "24.61382484436035",
         "10760.06640625",
         "0.0026279999874532223",
         "4.381972789764404",
         "349.3555908203125",
         "3.1327149868011475",
         "0.026193000376224518",
         "0.23290500044822693"
        ],
        [
         "2020-09-25 00:00:00",
         "0.09684500098228455",
         "4.712278842926025",
         "24.663387298583984",
         "10692.716796875",
         "0.002698000054806471",
         "4.42095422744751",
         "352.1832580566406",
         "3.1886119842529297",
         "0.027150999754667282",
         "0.24192200601100922"
        ],
        [
         "2020-09-26 00:00:00",
         "0.09556400030851364",
         "4.539248943328857",
         "26.221181869506836",
         "10750.7236328125",
         "0.002715999959036708",
         "4.347661972045898",
         "355.48809814453125",
         "3.1347029209136963",
         "0.027250999584794044",
         "0.24216000735759735"
        ]
       ],
       "shape": {
        "columns": 10,
        "rows": 5
       }
      },
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Ticker</th>\n",
       "      <th>ADA-USD</th>\n",
       "      <th>AVAX-USD</th>\n",
       "      <th>BNB-USD</th>\n",
       "      <th>BTC-USD</th>\n",
       "      <th>DOGE-USD</th>\n",
       "      <th>DOT-USD</th>\n",
       "      <th>ETH-USD</th>\n",
       "      <th>SOL-USD</th>\n",
       "      <th>TRX-USD</th>\n",
       "      <th>XRP-USD</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2020-09-22</th>\n",
       "      <td>0.081754</td>\n",
       "      <td>5.234632</td>\n",
       "      <td>24.100740</td>\n",
       "      <td>10538.459961</td>\n",
       "      <td>0.002656</td>\n",
       "      <td>4.174463</td>\n",
       "      <td>344.503174</td>\n",
       "      <td>2.915519</td>\n",
       "      <td>0.025044</td>\n",
       "      <td>0.233417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-23</th>\n",
       "      <td>0.076763</td>\n",
       "      <td>4.118469</td>\n",
       "      <td>22.856977</td>\n",
       "      <td>10246.186523</td>\n",
       "      <td>0.002560</td>\n",
       "      <td>4.031415</td>\n",
       "      <td>321.116302</td>\n",
       "      <td>2.836757</td>\n",
       "      <td>0.024859</td>\n",
       "      <td>0.221221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-24</th>\n",
       "      <td>0.082817</td>\n",
       "      <td>4.566561</td>\n",
       "      <td>24.613825</td>\n",
       "      <td>10760.066406</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>4.381973</td>\n",
       "      <td>349.355591</td>\n",
       "      <td>3.132715</td>\n",
       "      <td>0.026193</td>\n",
       "      <td>0.232905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-25</th>\n",
       "      <td>0.096845</td>\n",
       "      <td>4.712279</td>\n",
       "      <td>24.663387</td>\n",
       "      <td>10692.716797</td>\n",
       "      <td>0.002698</td>\n",
       "      <td>4.420954</td>\n",
       "      <td>352.183258</td>\n",
       "      <td>3.188612</td>\n",
       "      <td>0.027151</td>\n",
       "      <td>0.241922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-26</th>\n",
       "      <td>0.095564</td>\n",
       "      <td>4.539249</td>\n",
       "      <td>26.221182</td>\n",
       "      <td>10750.723633</td>\n",
       "      <td>0.002716</td>\n",
       "      <td>4.347662</td>\n",
       "      <td>355.488098</td>\n",
       "      <td>3.134703</td>\n",
       "      <td>0.027251</td>\n",
       "      <td>0.242160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Ticker       ADA-USD  AVAX-USD    BNB-USD       BTC-USD  DOGE-USD   DOT-USD  \\\n",
       "Date                                                                          \n",
       "2020-09-22  0.081754  5.234632  24.100740  10538.459961  0.002656  4.174463   \n",
       "2020-09-23  0.076763  4.118469  22.856977  10246.186523  0.002560  4.031415   \n",
       "2020-09-24  0.082817  4.566561  24.613825  10760.066406  0.002628  4.381973   \n",
       "2020-09-25  0.096845  4.712279  24.663387  10692.716797  0.002698  4.420954   \n",
       "2020-09-26  0.095564  4.539249  26.221182  10750.723633  0.002716  4.347662   \n",
       "\n",
       "Ticker         ETH-USD   SOL-USD   TRX-USD   XRP-USD  \n",
       "Date                                                  \n",
       "2020-09-22  344.503174  2.915519  0.025044  0.233417  \n",
       "2020-09-23  321.116302  2.836757  0.024859  0.221221  \n",
       "2020-09-24  349.355591  3.132715  0.026193  0.232905  \n",
       "2020-09-25  352.183258  3.188612  0.027151  0.241922  \n",
       "2020-09-26  355.488098  3.134703  0.027251  0.242160  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import necessary libraries\n",
    "import yfinance as yf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Set plotting style\n",
    "#plt.style.use('seaborn-darkgrid')\n",
    "# List of top 10 cryptocurrency tickers (as available on Yahoo Finance)\n",
    "tickers = ['BTC-USD', 'ETH-USD', 'BNB-USD', 'XRP-USD', 'ADA-USD',\n",
    "           'SOL-USD', 'DOGE-USD', 'DOT-USD', 'TRX-USD', 'AVAX-USD']\n",
    "\n",
    "# Fetch data from 2019-01-01 to present\n",
    "ydat = yf.download(tickers, start='2019-01-01', end='2024-06-01')\n",
    "data = ydat['Close']\n",
    "#data = ydat['Adj Close']\n",
    "\n",
    "# Drop rows with missing values\n",
    "data.dropna(inplace=True)\n",
    "\n",
    "# Display first few rows\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eef8836d",
   "metadata": {},
   "source": [
    "# Jupyter Notebook: Crypto Coin Price Trend Analysis and Outlier Detection\n",
    "\n",
    "## Overview\n",
    "This notebook analyzes price trends of top cryptocurrencies, quantifies similarities to a benchmark (BTC or average), and identifies outliers based on deviations. It follows the steps outlined in the query: data collection, preparation, normalization, benchmarking, and outlier detection.\n",
    "\n",
    "We use:\n",
    "- **Data Source**: CoinGecko API for historical daily closing prices (2019 to present).\n",
    "- **Coins**: BTC, ETH, BNB, XRP, ADA, SOL, DOGE, DOT, TRX, AVAX.\n",
    "- **Benchmark**: BTC as primary proxy; average of all coins as secondary.\n",
    "- **Analysis**: Daily log returns, cumulative returns, correlations, and deviation-based outlier detection.\n",
    "- **Visualization**: Plots of normalized cumulative returns to show \"identical paths\" and deviations.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7d5aecaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 1: Import Libraries\n",
    "import yfinance as yf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "from datetime import datetime\n",
    "import seaborn as sns\n",
    "\n",
    "## Step 2: Define Parameters\n",
    "# yfinance tickers\n",
    "tickers = ['BTC-USD', 'ETH-USD', 'BNB-USD', 'XRP-USD', 'ADA-USD', 'SOL-USD', 'DOGE-USD', 'DOT-USD', 'TRX-USD', 'AVAX-USD']\n",
    "\n",
    "# Date range\n",
    "start_date = '2019-01-01'\n",
    "end_date = datetime.now().strftime('%Y-%m-%d')\n",
    "\n",
    "# Function to fetch historical data from yfinance\n",
    "def fetch_coin_data(ticker):\n",
    "    try:\n",
    "        data = yf.download(ticker, start=start_date, end=end_date)\n",
    "        if data.empty:\n",
    "            print(f\"No data found for {ticker}\")\n",
    "            return None\n",
    "        #series = data['Adj Close']\n",
    "        series = data['Close']\n",
    "        series.name = ticker.split('-')[0].upper()  # e.g., 'BTC-USD' -> 'BTC'\n",
    "        return series\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching data for {ticker}: {e}\")\n",
    "        return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fec49653",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "/tmp/ipykernel_2528194/330105761.py:21: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
      "  data = yf.download(ticker, start=start_date, end=end_date)\n",
      "[*********************100%***********************]  1 of 1 completed"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetched Data Preview:\n",
      "                    BTC         ETH       BNB       XRP       ADA  SOL  \\\n",
      "Date                                                                     \n",
      "2019-01-01  3843.520020  140.819412  6.075273  0.364771  0.042547  NaN   \n",
      "2019-01-02  3943.409424  155.047684  6.188613  0.375243  0.045258  NaN   \n",
      "2019-01-03  3836.741211  149.135010  5.903535  0.360224  0.042682  NaN   \n",
      "2019-01-04  3857.717529  154.581940  6.065138  0.356747  0.043812  NaN   \n",
      "2019-01-05  3845.194580  155.638596  6.065543  0.355275  0.044701  NaN   \n",
      "\n",
      "                DOGE  DOT       TRX  AVAX  \n",
      "Date                                       \n",
      "2019-01-01  0.002392  NaN  0.019561   NaN  \n",
      "2019-01-02  0.002407  NaN  0.020222   NaN  \n",
      "2019-01-03  0.002360  NaN  0.020088   NaN  \n",
      "2019-01-04  0.002315  NaN  0.021492   NaN  \n",
      "2019-01-05  0.002319  NaN  0.022294   NaN  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Step 3: Collect and Prepare Data\n",
    "# Build prices_df incrementally to avoid dict-of-scalars error\n",
    "prices_df = pd.DataFrame()\n",
    "for ticker in tickers:\n",
    "    data = fetch_coin_data(ticker)\n",
    "    if data is not None:\n",
    "        prices_df[data.name] = data\n",
    "\n",
    "prices_df = prices_df.dropna(axis=1, how='all')  # Drop any failed fetches\n",
    "if prices_df.empty:\n",
    "    raise ValueError(\"No data fetched for any coins. Check tickers, dates, or internet connection.\")\n",
    "print(\"Fetched Data Preview:\")\n",
    "print(prices_df.head())\n",
    "\n",
    "# Calculate daily log returns\n",
    "returns_df = np.log(prices_df / prices_df.shift(1))\n",
    "returns_df = returns_df.dropna()  # Remove NaNs from first row\n",
    "\n",
    "# Normalize returns for visualization (min-max scaling to [0,1])\n",
    "normalized_returns = (returns_df - returns_df.min()) / (returns_df.max() - returns_df.min())\n",
    "\n",
    "# Calculate cumulative returns for trend analysis\n",
    "cum_returns = (1 + returns_df).cumprod()\n",
    "normalized_cum_returns = (cum_returns - cum_returns.min()) / (cum_returns.max() - cum_returns.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7b2b4bf1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benchmark (BTC) Cumulative Returns Preview:\n",
      "Date\n",
      "2020-09-23    0.971874\n",
      "2020-09-24    1.019434\n",
      "2020-09-25    1.013033\n",
      "2020-09-26    1.018514\n",
      "2020-09-27    1.020837\n",
      "Name: BTC, dtype: float64\n",
      "\n",
      "Correlations with Benchmark (BTC):\n",
      "BTC     1.000000\n",
      "ETH     0.807628\n",
      "DOT     0.690906\n",
      "ADA     0.661981\n",
      "BNB     0.652579\n",
      "AVAX    0.610340\n",
      "SOL     0.580406\n",
      "XRP     0.555456\n",
      "TRX     0.533154\n",
      "DOGE    0.515730\n",
      "dtype: float64\n",
      "\n",
      "Outlier Flags (Proportion of Time Deviating >2 Std Devs):\n",
      "BNB     0.601795\n",
      "SOL     0.058890\n",
      "DOGE    0.038138\n",
      "ADA     0.027482\n",
      "AVAX    0.009534\n",
      "XRP     0.001122\n",
      "ETH     0.000000\n",
      "BTC     0.000000\n",
      "DOT     0.000000\n",
      "TRX     0.000000\n",
      "dtype: float64\n",
      "\n",
      "Identified Outliers (High Deviation Coins):\n",
      "BNB    0.601795\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 365416 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 135196 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 114060 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Step 4: Define Benchmark 'Normal' Trend\n",
    "# Primary benchmark: BTC as proxy\n",
    "benchmark_coin = 'BTC'\n",
    "benchmark_returns = returns_df[benchmark_coin]\n",
    "benchmark_cum_returns = cum_returns[benchmark_coin]\n",
    "\n",
    "# Secondary benchmark: Average daily return of all coins\n",
    "average_returns = returns_df.mean(axis=1)\n",
    "average_cum_returns = (1 + average_returns).cumprod()\n",
    "\n",
    "print(f\"Benchmark ({benchmark_coin}) Cumulative Returns Preview:\")\n",
    "print(benchmark_cum_returns.head())\n",
    "\n",
    "## Step 5: Quantify Adherence and Identify Outliers\n",
    "# Calculate correlations with benchmark (Pearson correlation on returns)\n",
    "correlations = returns_df.corrwith(benchmark_returns)\n",
    "print(\"\\nCorrelations with Benchmark (BTC):\")\n",
    "print(correlations.sort_values(ascending=False))\n",
    "\n",
    "# Calculate deviations: Difference in cumulative returns from benchmark\n",
    "deviations = cum_returns.subtract(benchmark_cum_returns, axis=0)\n",
    "\n",
    "# Identify outliers: Coins where absolute deviation exceeds 2 std devs from mean deviation\n",
    "threshold = 2  # Z-score threshold for outliers\n",
    "\n",
    "# Custom function to compute row-wise z-scores handling NaNs and zero std\n",
    "def row_zscore(row):\n",
    "    mean = np.nanmean(row)\n",
    "    std = np.nanstd(row)\n",
    "    if std == 0 or np.isnan(std):\n",
    "        return pd.Series(np.nan, index=row.index)\n",
    "    return (row - mean) / std\n",
    "\n",
    "# Compute z-scores for deviations (preserves shape, handles NaNs)\n",
    "z_scores = deviations.apply(row_zscore, axis=1)\n",
    "\n",
    "# Flag outliers: Proportion of time |z-score| > threshold (over available data)\n",
    "outlier_flags = ((z_scores.abs() > threshold) & z_scores.notna()).sum(axis=0) / z_scores.notna().sum(axis=0)\n",
    "outliers = outlier_flags[outlier_flags > 0.1]  # Flag if >10% of the time (adjustable)\n",
    "\n",
    "print(\"\\nOutlier Flags (Proportion of Time Deviating >2 Std Devs):\")\n",
    "print(outlier_flags.sort_values(ascending=False))\n",
    "print(\"\\nIdentified Outliers (High Deviation Coins):\")\n",
    "print(outliers)\n",
    "\n",
    "## Step 6: Visualize Trends and Deviations\n",
    "# Plot 1: Normalized Cumulative Returns (to show similar paths)\n",
    "plt.figure(figsize=(14, 7))\n",
    "for coin in normalized_cum_returns.columns:\n",
    "    plt.plot(normalized_cum_returns.index, normalized_cum_returns[coin], label=coin)\n",
    "plt.plot(normalized_cum_returns.index, normalized_cum_returns[benchmark_coin], label=f'Benchmark ({benchmark_coin})', linewidth=3, color='black', linestyle='--')\n",
    "plt.title('Normalized Cumulative Returns: Similar Trends Across Coins')\n",
    "plt.xlabel('Date')\n",
    "plt.ylabel('Normalized Cumulative Return')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "# Plot 2: Deviations from Benchmark\n",
    "plt.figure(figsize=(14, 7))\n",
    "for coin in deviations.columns:\n",
    "    color = 'red' if coin in outliers.index else 'blue'\n",
    "    plt.plot(deviations.index, deviations[coin], label=coin, color=color)\n",
    "plt.axhline(0, color='black', linestyle='--')\n",
    "plt.title(f'Deviations from Benchmark ({benchmark_coin}) Cumulative Returns')\n",
    "plt.xlabel('Date')\n",
    "plt.ylabel('Deviation')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "# Plot 3: Correlation Heatmap (Optional, for deeper insight)\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(returns_df.corr(), annot=True, cmap='coolwarm')\n",
    "plt.title('Correlation Matrix of Coin Returns')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "248eee97",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2528194/3241911784.py:10: FutureWarning: The default fill_method='pad' in DataFrame.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n",
      "  sma_df = prices_df.pct_change().rolling(window=sma_window).mean()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "50-Day SMA Preview:\n",
      "            BTC  ETH  BNB  XRP  ADA  SOL  DOGE  DOT  TRX  AVAX\n",
      "Date                                                          \n",
      "2019-01-01  NaN  NaN  NaN  NaN  NaN  NaN   NaN  NaN  NaN   NaN\n",
      "2019-01-02  NaN  NaN  NaN  NaN  NaN  NaN   NaN  NaN  NaN   NaN\n",
      "2019-01-03  NaN  NaN  NaN  NaN  NaN  NaN   NaN  NaN  NaN   NaN\n",
      "2019-01-04  NaN  NaN  NaN  NaN  NaN  NaN   NaN  NaN  NaN   NaN\n",
      "2019-01-05  NaN  NaN  NaN  NaN  NaN  NaN   NaN  NaN  NaN   NaN\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 435408 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "K-Means Clusters (0: Trend-Followers, 1: Deviators?):\n",
      "BTC     1\n",
      "ETH     1\n",
      "BNB     1\n",
      "XRP     0\n",
      "ADA     1\n",
      "SOL     1\n",
      "DOGE    1\n",
      "DOT     1\n",
      "TRX     1\n",
      "AVAX    1\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# 4.1: Moving Averages (50-day SMA)\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "sma_window = 50\n",
    "sma_df = prices_df.pct_change().rolling(window=sma_window).mean()\n",
    "sma_df = sma_df.apply(lambda x: (x - x.min()) / (x.max() - x.min()))  # Normalize returns\n",
    "print(\"\\n50-Day SMA Preview:\")\n",
    "print(sma_df.head())\n",
    "\n",
    "# Plot SMA for comparison\n",
    "plt.figure(figsize=(14, 7))\n",
    "for coin in sma_df.columns:\n",
    "    plt.plot(sma_df.index, sma_df[coin], label=coin)\n",
    "plt.title('50-Day Simple Moving Averages (SMA) of Normalized Returns for Coins')\n",
    "plt.xlabel('Date')\n",
    "plt.ylabel('Normalized Return')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "# 4.2: Clustering (K-Means on returns)\n",
    "# Prepare data: Fill NaNs, transpose for clustering (coins as samples, days as features)\n",
    "\n",
    "# Backtest parameters\n",
    "window_size = 365  # 1-year rolling windows (adjustable, e.g., 90 for quarterly)\n",
    "outlier_threshold = 2.0  # Z-score threshold\n",
    "cluster_n = 2  # For K-Means: trend-followers vs. deviators\n",
    "\n",
    "\n",
    "clustering_data = returns_df.fillna(0).T  # Transpose: rows=coins, columns=days\n",
    "\n",
    "kmeans = KMeans(n_clusters=cluster_n, random_state=42)\n",
    "clusters = kmeans.fit_predict(clustering_data)\n",
    "cluster_labels = pd.Series(clusters, index=clustering_data.index)\n",
    "print(\"\\nK-Means Clusters (0: Trend-Followers, 1: Deviators?):\")\n",
    "print(cluster_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "84ea6724",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Regression Betas (Alignment to BTC):\n",
      "DOGE    1.220616\n",
      "AVAX    1.207550\n",
      "DOT     1.151600\n",
      "SOL     1.147277\n",
      "ADA     1.085205\n",
      "ETH     1.071199\n",
      "XRP     0.992635\n",
      "BNB     0.896157\n",
      "TRX     0.720788\n",
      "dtype: float64\n",
      "\n",
      "Residual-Based Outlier Proportions:\n",
      "ETH     0.057768\n",
      "SOL     0.057768\n",
      "ADA     0.051598\n",
      "DOT     0.047112\n",
      "XRP     0.044868\n",
      "AVAX    0.042064\n",
      "TRX     0.040942\n",
      "BNB     0.039821\n",
      "DOGE    0.032529\n",
      "dtype: float64\n",
      "\n",
      "Isolation Forest Outliers (Sample):\n",
      "Normal     1604\n",
      "Outlier     179\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "## Step 6: Advanced Outlier Detection Methods\n",
    "# 6.1: Residual Analysis (Linear Regression)\n",
    "residuals_dict = {}\n",
    "betas = {}\n",
    "for coin in returns_df.columns:\n",
    "    if coin == benchmark_coin:\n",
    "        continue\n",
    "    X = benchmark_returns.values.reshape(-1, 1)\n",
    "    y = returns_df[coin].values\n",
    "    model = LinearRegression().fit(X, y)\n",
    "    preds = model.predict(X)\n",
    "    residuals = y - preds\n",
    "    residuals_dict[coin] = residuals\n",
    "    betas[coin] = model.coef_[0]  # Beta against benchmark\n",
    "\n",
    "residuals_df = pd.DataFrame(residuals_dict, index=returns_df.index)\n",
    "print(\"\\nRegression Betas (Alignment to BTC):\")\n",
    "print(pd.Series(betas).sort_values(ascending=False))\n",
    "\n",
    "# Flag high residuals (e.g., |residual| > 2 * std)\n",
    "residual_outliers = (residuals_df.abs() > 2 * residuals_df.std()).mean(axis=0)\n",
    "print(\"\\nResidual-Based Outlier Proportions:\")\n",
    "print(residual_outliers.sort_values(ascending=False))\n",
    "\n",
    "# 6.2: Machine Learning - Isolation Forest\n",
    "# Prepare data: Fill NaNs, use returns\n",
    "ml_data = returns_df.fillna(0)\n",
    "iso_forest = IsolationForest(contamination=0.1, random_state=42)\n",
    "outlier_preds = iso_forest.fit_predict(ml_data)\n",
    "outlier_scores = iso_forest.decision_function(ml_data)\n",
    "ml_outliers = pd.Series(outlier_preds, index=ml_data.index).apply(lambda x: 'Outlier' if x == -1 else 'Normal')\n",
    "print(\"\\nIsolation Forest Outliers (Sample):\")\n",
    "print(ml_outliers.value_counts())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "67c5818f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Backtest Summary (Sample Windows):\n",
      "Window 2020-09-23 to 2021-09-23: Outlier Freq=0.20, Avg Magnitude=2.41, Outliers=['ADA', 'DOGE']\n",
      "Window 2020-10-23 to 2021-10-23: Outlier Freq=0.20, Avg Magnitude=2.70, Outliers=['SOL', 'DOGE']\n",
      "Window 2020-11-22 to 2021-11-22: Outlier Freq=0.20, Avg Magnitude=2.77, Outliers=['SOL', 'DOGE']\n",
      "Window 2020-12-22 to 2021-12-22: Outlier Freq=0.10, Avg Magnitude=3.57, Outliers=['SOL']\n",
      "Window 2021-01-21 to 2022-01-21: Outlier Freq=0.20, Avg Magnitude=2.00, Outliers=['BNB', 'SOL']\n",
      "\n",
      "Backtest Metrics:\n",
      "Accuracy (Proxy): 0.08\n",
      "Average Strategy Return: -0.0011\n",
      "Sharpe Ratio: -0.40\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 57236 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Step 7: Backtest Setup with Rolling Windows\n",
    "# Divide data into rolling windows (e.g., 1-year)\n",
    "alignments = []  # List to store per-window alignments\n",
    "outlier_metrics = []  # List to store per-window outlier data\n",
    "strategy_returns = []  # For hypothetical strategy\n",
    "\n",
    "dates = returns_df.index\n",
    "step = 30  # Step size for rolling (e.g., monthly shifts)\n",
    "for i in range(0, len(dates) - window_size, step):\n",
    "    window_start = dates[i]\n",
    "    window_end = dates[i + window_size]\n",
    "    window_returns = returns_df.loc[window_start:window_end]\n",
    "    if window_returns.empty:\n",
    "        continue\n",
    "    \n",
    "    # Benchmark: Median returns (robust to outliers)\n",
    "    benchmark_median = window_returns.median(axis=1)\n",
    "    benchmark_cum = (1 + benchmark_median).cumprod()\n",
    "    \n",
    "    # Measure alignment: Cosine similarity and regression beta\n",
    "    window_alignments = {}\n",
    "    for coin in window_returns.columns:\n",
    "        # Cosine similarity\n",
    "        sim = cosine_similarity(window_returns[coin].values.reshape(1, -1), benchmark_median.values.reshape(1, -1))[0][0]\n",
    "        # Beta\n",
    "        X = benchmark_median.values.reshape(-1, 1)\n",
    "        y = window_returns[coin].values\n",
    "        beta = LinearRegression().fit(X, y).coef_[0]\n",
    "        window_alignments[coin] = {'cosine_sim': sim, 'beta': beta}\n",
    "    alignments.append((window_start, window_end, window_alignments))\n",
    "    \n",
    "    # Identify outliers in window (Z-score on deviations)\n",
    "    window_cum = (1 + window_returns).cumprod()\n",
    "    window_dev = window_cum.subtract(benchmark_cum, axis=0)\n",
    "    window_z = window_dev.apply(row_zscore, axis=1)\n",
    "    window_flags = ((window_z.abs() > outlier_threshold) & window_z.notna()).sum(axis=0) / window_z.notna().sum(axis=0)\n",
    "    window_outliers = window_flags[window_flags > 0.1]\n",
    "    \n",
    "    # Track metrics: Frequency, magnitude\n",
    "    freq = len(window_outliers) / len(window_returns.columns)\n",
    "    magnitude = window_dev.abs().mean().mean() if not window_dev.empty else 0\n",
    "    outlier_metrics.append((window_start, window_end, freq, magnitude, window_outliers))\n",
    "    \n",
    "    # Hypothetical strategy: Short outliers (assume -1 * their next period return if flagged)\n",
    "    next_start = min(i + window_size, len(dates) - 1)\n",
    "    next_end = min(next_start + step, len(dates) - 1)\n",
    "    if next_end > next_start:\n",
    "        next_returns = returns_df.iloc[next_start:next_end]\n",
    "        strat_ret = 0\n",
    "        for coin in window_outliers.index:\n",
    "            if coin in next_returns.columns:\n",
    "                strat_ret -= next_returns[coin].mean()  # Short the outlier\n",
    "        strategy_returns.append(strat_ret / len(window_outliers) if len(window_outliers) > 0 else 0)\n",
    "\n",
    "# Summarize backtest\n",
    "print(\"\\nBacktest Summary (Sample Windows):\")\n",
    "for start, end, freq, mag, outs in outlier_metrics[:5]:  # First 5 for brevity\n",
    "    print(f\"Window {start.date()} to {end.date()}: Outlier Freq={freq:.2f}, Avg Magnitude={mag:.2f}, Outliers={outs.index.tolist()}\")\n",
    "\n",
    "# Performance Metrics\n",
    "# Accuracy: Proportion of flagged outliers with high deviation magnitude (e.g., > mean + 2SD)\n",
    "all_magnitudes = [mag for _, _, _, mag, _ in outlier_metrics]\n",
    "mag_mean = np.mean(all_magnitudes)\n",
    "mag_std = np.std(all_magnitudes)\n",
    "accuracy = np.mean([1 if mag > mag_mean + 2 * mag_std else 0 for mag in all_magnitudes])  # Simple proxy\n",
    "\n",
    "# Strategy Returns and Sharpe\n",
    "strategy_series = pd.Series(strategy_returns)\n",
    "cum_strategy = (1 + strategy_series).cumprod()\n",
    "sharpe = (strategy_series.mean() / strategy_series.std()) * np.sqrt(12) if strategy_series.std() != 0 else 0  # Annualized\n",
    "\n",
    "print(f\"\\nBacktest Metrics:\\nAccuracy (Proxy): {accuracy:.2f}\\nAverage Strategy Return: {strategy_series.mean():.4f}\\nSharpe Ratio: {sharpe:.2f}\")\n",
    "\n",
    "# Plot Hypothetical Strategy Cumulative Returns\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(range(len(cum_strategy)), cum_strategy, label='Strategy Cumulative Returns')\n",
    "plt.title('Hypothetical Strategy: Short Outliers')\n",
    "plt.xlabel('Window Index')\n",
    "plt.ylabel('Cumulative Return')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "## Conclusion\n",
    "# - Trends detected via SMA and clustering.\n",
    "# - Outliers identified using multiple methods.\n",
    "# - Backtest simulates rolling windows, measures alignments, and evaluates a simple strategy.\n",
    "# - Adjust parameters (e.g., window_size, threshold) for refinement."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
