{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f9507824",
   "metadata": {},
   "source": [
    "# Saving Paired Data\n",
    "\n",
    "First let's just import the driver."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3d43faf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please install h5py to open files from the Amazon S3 servers.\n",
      "Please install h5netcdf to open files from the Amazon S3 servers.\n"
     ]
    }
   ],
   "source": [
    "from melodies_monet import driver"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1543badf-885c-44ca-8478-aaaa4ac84485",
   "metadata": {},
   "source": [
    "## Read model, obs and Pair the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "65671ca7",
   "metadata": {
    "tags": [
     "hide-output"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "example:wrfchem:racm_esrl\n",
      "**** Reading WRF-Chem model output...\n",
      "example:wrfchem:racm_esrl_vcp\n",
      "**** Reading WRF-Chem model output...\n",
      "After pairing:                         time  BARPR   BC   CO   NO  NO2  NO2Y  NOX  NOY  OZONE  \\\n",
      "0       2019-09-01 00:00:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0   25.0   \n",
      "1       2019-09-01 00:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "2       2019-09-01 00:30:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "3       2019-09-01 01:00:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0   24.0   \n",
      "4       2019-09-01 01:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "...                     ...    ...  ...  ...  ...  ...   ...  ...  ...    ...   \n",
      "7916521 2019-09-29 23:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "7916522 2019-09-29 23:30:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "7916523 2019-09-30 00:00:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    8.0   \n",
      "7916524 2019-09-30 00:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "7916525 2019-09-30 00:30:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "\n",
      "         ...  longitude  cmsa_name  msa_code  msa_name  state_name  \\\n",
      "0        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "1        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "2        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "3        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "4        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "...      ...        ...        ...       ...       ...         ...   \n",
      "7916521  ...    69.2725       -1.0      -1.0                         \n",
      "7916522  ...    69.2725       -1.0      -1.0                         \n",
      "7916523  ...    69.2725       -1.0      -1.0                         \n",
      "7916524  ...    69.2725       -1.0      -1.0                         \n",
      "7916525  ...    69.2725       -1.0      -1.0                         \n",
      "\n",
      "         epa_region          time_local     siteid  PM2_5_DRY  o3  \n",
      "0                CA 2019-08-31 20:00:00  000010102        NaN NaN  \n",
      "1                CA 2019-08-31 20:15:00  000010102        NaN NaN  \n",
      "2                CA 2019-08-31 20:30:00  000010102        NaN NaN  \n",
      "3                CA 2019-08-31 21:00:00  000010102        NaN NaN  \n",
      "4                CA 2019-08-31 21:15:00  000010102        NaN NaN  \n",
      "...             ...                 ...        ...        ...  ..  \n",
      "7916521        DSUZ 2019-09-30 04:15:00  UZB010001        NaN NaN  \n",
      "7916522        DSUZ 2019-09-30 04:30:00  UZB010001        NaN NaN  \n",
      "7916523        DSUZ 2019-09-30 05:00:00  UZB010001        NaN NaN  \n",
      "7916524        DSUZ 2019-09-30 05:15:00  UZB010001        NaN NaN  \n",
      "7916525        DSUZ 2019-09-30 05:30:00  UZB010001        NaN NaN  \n",
      "\n",
      "[7916526 rows x 36 columns]\n",
      "After pairing:                         time  BARPR   BC   CO   NO  NO2  NO2Y  NOX  NOY  OZONE  \\\n",
      "0       2019-09-01 00:00:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0   25.0   \n",
      "1       2019-09-01 00:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "2       2019-09-01 00:30:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "3       2019-09-01 01:00:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0   24.0   \n",
      "4       2019-09-01 01:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "...                     ...    ...  ...  ...  ...  ...   ...  ...  ...    ...   \n",
      "7916521 2019-09-29 23:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "7916522 2019-09-29 23:30:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "7916523 2019-09-30 00:00:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    8.0   \n",
      "7916524 2019-09-30 00:15:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "7916525 2019-09-30 00:30:00   -1.0 -1.0 -1.0 -1.0 -1.0  -1.0 -1.0 -1.0    NaN   \n",
      "\n",
      "         ...  longitude  cmsa_name  msa_code  msa_name  state_name  \\\n",
      "0        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "1        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "2        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "3        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "4        ...   -52.8167       -1.0      -1.0                    CC   \n",
      "...      ...        ...        ...       ...       ...         ...   \n",
      "7916521  ...    69.2725       -1.0      -1.0                         \n",
      "7916522  ...    69.2725       -1.0      -1.0                         \n",
      "7916523  ...    69.2725       -1.0      -1.0                         \n",
      "7916524  ...    69.2725       -1.0      -1.0                         \n",
      "7916525  ...    69.2725       -1.0      -1.0                         \n",
      "\n",
      "         epa_region          time_local     siteid  PM2_5_DRY  o3  \n",
      "0                CA 2019-08-31 20:00:00  000010102        NaN NaN  \n",
      "1                CA 2019-08-31 20:15:00  000010102        NaN NaN  \n",
      "2                CA 2019-08-31 20:30:00  000010102        NaN NaN  \n",
      "3                CA 2019-08-31 21:00:00  000010102        NaN NaN  \n",
      "4                CA 2019-08-31 21:15:00  000010102        NaN NaN  \n",
      "...             ...                 ...        ...        ...  ..  \n",
      "7916521        DSUZ 2019-09-30 04:15:00  UZB010001        NaN NaN  \n",
      "7916522        DSUZ 2019-09-30 04:30:00  UZB010001        NaN NaN  \n",
      "7916523        DSUZ 2019-09-30 05:00:00  UZB010001        NaN NaN  \n",
      "7916524        DSUZ 2019-09-30 05:15:00  UZB010001        NaN NaN  \n",
      "7916525        DSUZ 2019-09-30 05:30:00  UZB010001        NaN NaN  \n",
      "\n",
      "[7916526 rows x 36 columns]\n"
     ]
    }
   ],
   "source": [
    "an = driver.analysis()\n",
    "an.control = \"control_wrfchem_saveandread.yaml\"\n",
    "an.read_control()\n",
    "an.control_dict\n",
    "\n",
    "an.open_models()\n",
    "an.open_obs()\n",
    "an.pair_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddc902a4-7885-4c3e-b820-7096d00dddc0",
   "metadata": {},
   "source": [
    "## Save data using control file\n",
    "\n",
    "````{admonition} Note: This is the complete file that was loaded.\n",
    ":class: dropdown\n",
    "\n",
    "```{literalinclude} control_wrfchem_saveandread.yaml\n",
    ":caption:\n",
    ":linenos:\n",
    "```\n",
    "````\n",
    "\n",
    "The driver will save the data based on the information included in {doc}`the control file </appendix/yaml>` by calling {func}`an.save_analysis()<melodies_monet.driver.analysis.save_analysis>`.\n",
    "\n",
    "In the control file analysis section, setting method to `'netcdf'` for a given attribute of the analysis class (e.g., paired,  models, obs) will write netcdf4 files to the output directory. For example, when saving out paired data, it will write a separate file for each model/obs pairing. The filenames take the format `<prefix>_<label>.nc4`, where for example the label of a paired class may be `'airnow_RACM_ESRL'` or `'airnow_RACM_ESRL_VCP'`.\n",
    "\n",
    "In the control file analysis section, setting method to `'pkl'` for a given attribute of the analysis class (e.g., paired, models, obs) will write [pickle files](https://joblib.readthedocs.io/en/latest/generated/joblib.dump.html) to the output directory. Unlike with the netCDF files, all pairs will be saved in the same pickle file. The output filename is set with the `'output_name'` in the control file.  \n",
    "\n",
    "```{note}\n",
    "Be careful when saving pickle files for later analysis or when files will be used by multiple users. A change to the structure of xarray objects between saving the file and reading the file (for example if the version of xarray is different) can break the functionality of reading saved pickle files with MELODIES-MONET.\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94e9281e-0845-411b-a59d-96456ccb5a6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing: ./output/save_and_read/0905_airnow_RACM_ESRL.nc4\n",
      "Writing: ./output/save_and_read/0905_airnow_RACM_ESRL_VCP.nc4\n"
     ]
    }
   ],
   "source": [
    "an.save_analysis()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72514743-b2cb-453c-9ff6-7e4873a90b20",
   "metadata": {},
   "source": [
    "## Save data without using control file\n",
    "\n",
    "Alternatively, the same can be achieved by calling {func}`~melodies_monet.util.write_util.write_analysis_ncf` or {func}`~melodies_monet.util.write_util.write_pkl` directly. The object to save must be an attribute of the instance of the analysis class (e.g., {attr}`an.paired <melodies_monet.driver.analysis.paired>`, {attr}`an.models <melodies_monet.driver.analysis.models>`, {attr}`an.obs <melodies_monet.driver.analysis.obs>`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "87601ffc-a12c-4a4c-afc2-5168a90f1c6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing: ./output/save_and_read/0905_airnow_RACM_ESRL.nc4\n",
      "Writing: ./output/save_and_read/0905_airnow_RACM_ESRL_VCP.nc4\n"
     ]
    }
   ],
   "source": [
    "# For netCDF files \n",
    "from melodies_monet.util.write_util import write_analysis_ncf\n",
    "write_analysis_ncf(obj=an.paired, output_dir='./output/save_and_read',\n",
    "                                               fn_prefix='0905')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cc4c619e-5064-430a-9597-df8610a83d41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing: ./output/save_and_read/0905.pkl\n"
     ]
    }
   ],
   "source": [
    "# For pickle files \n",
    "from melodies_monet.util.write_util import write_pkl\n",
    "write_pkl(obj=an.paired, output_name='./output/save_and_read/0905.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}