reworked to use single glider netcdf from bastiens toolbox

notebooks/explore_data.ipynb

@@ -23,7 +23,7 @@
     "import copy\n",
     "from netCDF4 import Dataset\n",
     "from pathlib import Path\n",
-    "from datetime import datetime\n",
+    "from datetime import datetime, timedelta\n",
     "notebook_dir = !pwd # Replace with !echo %cd% if using windows\n",
     "library_dir = Path(notebook_dir[0]).parent.absolute()\n",
     "sys.path.append(str(library_dir))\n",
@@ -72,7 +72,7 @@
      "text": [
       "/home/callum/anaconda3/envs/adcp-glider/lib/python3.7/site-packages/numpy/lib/nanfunctions.py:1115: RuntimeWarning: All-NaN slice encountered\n",
       "  overwrite_input=overwrite_input)\n",
-      "/media/callum/storage/Documents/adcp-glider/src/data/adcp_mission_analysis.py:374: RuntimeWarning: Mean of empty slice\n",
+      "/media/callum/storage/Documents/adcp-glider/src/data/adcp_mission_analysis.py:364: RuntimeWarning: Mean of empty slice\n",
       "  vel_referenced = vel - np.tile(np.nanmean(vel,0) - ref_vel,(len(bin_centers),1))\n"
      ]
     }
@@ -103,6 +103,32 @@
     "# overview.join(beam_attrs)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "adcp_df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Check that the data start when you expect. The first file in particular may have data from a previous sim dive or bench test. adapt the following cell to select the data starting on the correct day"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# For this dataset, need to remove the data before 23 Jan they are from 2019 benchtests and 2020 sim dives\n",
+    "adcp_df = adcp_df['2020-01-23':]"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -164,7 +190,7 @@
    },
    "outputs": [],
    "source": [
-    "glider_df = read_glider_nc(glider_raw_path)\n",
+    "glider_df = read_glider_nc(glider_raw_path / 'sg637_full_timeseries.nc')\n",
     "glider_df"
    ]
   },
@@ -180,13 +206,13 @@
    "source": [
     "fig, ax = plt.subplots(3,1,figsize=(20,26), sharex=True)\n",
     "ax = ax.ravel()\n",
-    "ax[0].scatter(glider_df.index, glider_df.pitch_gl, label='glider')\n",
+    "ax[0].scatter(glider_df.index, glider_df.pitch, label='glider')\n",
     "ax[0].scatter(adcp_df.index, adcp_df.pitch_ad, label='adcp')\n",
     "ax[0].set(xlim=[datetime(2020,1,24,0), datetime(2020,1,24,12)], ylim=[-30,30], ylabel='Pitch')\n",
-    "ax[1].scatter(glider_df.index, glider_df.roll_gl)\n",
+    "ax[1].scatter(glider_df.index, glider_df.roll)\n",
     "ax[1].scatter(adcp_df.index, adcp_df.roll_ad)\n",
     "ax[1].set(ylim=[-20,20], ylabel='Roll')\n",
-    "ax[2].scatter(glider_df.index, glider_df.heading_gl)\n",
+    "ax[2].scatter(glider_df.index, glider_df.heading)\n",
     "ax[2].scatter(adcp_df.index, adcp_df.heading_ad)\n",
     "ax[2].set(ylim=[50,200], ylabel='Heading')\n",
     "ax[0].legend();\n",
@@ -204,7 +230,7 @@
    "outputs": [],
    "source": [
     "fig, ax = plt.subplots(figsize=(10,6))\n",
-    "ax.scatter(glider_df.index, glider_df.pressure_gl, label='glider', s=1)\n",
+    "ax.scatter(glider_df.index, glider_df.pressure, label='glider', s=1)\n",
     "ax.scatter(adcp_df.index, adcp_df.pressure_ad, label='adcp', s=1)\n",
     "ax.set(xlim=[datetime(2020,1,24,0), datetime(2020,1,24,12)],ylim = [-10,750], ylabel='Pressure (dbar)')\n",
     "ax.legend()\n",
@@ -391,6 +417,21 @@
     "ax[1].pcolor(all_the_shear[101:150,:,1], cmap='RdBu_r', vmin=-0.5, vmax=0.5)\n",
     "ax[2].pcolor(all_the_shear[101:150,:,2], cmap='RdBu_r', vmin=-0.5, vmax=0.5)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "-------------------\n",
+    "# Rough stuff"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

src/data/adcp_mission_analysis.py

@@ -72,7 +72,7 @@ def centretoedge(x_in):
 
 
 @dataclass
-class adcp_profile:
+class AdcpProfile:
     """
     Class object to store data for a single adcp profile
     """
@@ -103,34 +103,23 @@ class adcp_profile:
     amp_binned: float
 
 
-def read_glider_nc(glider_dir):
-    yos_path = glider_dir.rglob("p*.nc")
-    yos_list = []
-    for path in yos_path:
-        yos_list.append(str(path))
-    nets = np.sort(yos_list)
-    dive_nums, pressure, time, roll, pitch, heading = [], [], [], [], [], []
-
-    for net in nets:
-        nc = xr.open_dataset(net)
-        dive_num = net[-7:-3]
-        dive_nums = dive_nums + list(np.tile(dive_num, (1, len(nc.pressure)))[0])
-        time = time + list(nc.time.data)
-        pressure = pressure + list(nc.pressure.data)
-        roll = roll + list(nc.eng_rollAng.data)
-        pitch = pitch + list(nc.eng_pitchAng.data)
-        heading = heading + list(nc.eng_head.data)
-        temp = nc.temperature.data
-        conductivity = nc.conductivity.data
-        lon = nc.longitude
-        lat = nc.latitude
-    glider_df = pd.DataFrame(
-        {'dive_num': dive_nums, 'pressure_gl': pressure, 'pitch_gl': pitch, 'roll_gl': roll, 'heading_gl': heading},
-        index=pd.to_datetime(time))
-    return glider_df
+def glidertimetoneat(glider_time):
+    time_cont_blank = np.empty(len(glider_time), dtype=datetime)
+    timestamp = []
+    for i in range(len(glider_time)):
+        day = datetime.fromordinal(int(glider_time[i]))
+        dayfrac = timedelta(days=glider_time[i]%1) - timedelta(days = 366)
+        time_cont_blank[i] = day + dayfrac
+        timestamp.append(pd.Timestamp(time_cont_blank[i]))
+    return time_cont_blank, timestamp
 
 
-glider_df = read_glider_nc(Path('/home/callum/Documents/Eureka/data/glider-nc-transfer/full-nc'))
+def read_glider_nc(glider_netcdf_file):
+    glider_nc = xr.open_dataset(glider_netcdf_file)
+    df = glider_nc.to_dataframe()
+    df.rename(columns={'unnamed': 'roll', 'unnamed1': 'pitch', 'unnamed2': 'heading'}, inplace=True)
+    df['glider_time'], df.index = glidertimetoneat(df.index)
+    return df
 
 
 def adcp_import_data(working_dir):
@@ -248,9 +237,9 @@ def adcp_import_data(working_dir):
         amp_binned, __, __ = bin_attr(amp_flag, measurement_z)
         ad2cp_dict = data_av.variables
 
-        profile = adcp_profile(str(index), time, cell_center, pitch, roll, heading, cor_beam, amp_beam, vel_beam,
-                               vel_xyz, vel_enu, beam_miss, flag_bad_data, shear_one_cell, shear_binned, vels_in_bin, vel_referenced,
-                               vel_z, beam_number, pressure, glider_z, measurement_z, glider_w_from_p, ad2cp_dict, amp_binned)
+        profile = AdcpProfile(str(index), time, cell_center, pitch, roll, heading, cor_beam, amp_beam, vel_beam,
+                              vel_xyz, vel_enu, beam_miss, flag_bad_data, shear_one_cell, shear_binned, vels_in_bin, vel_referenced,
+                              vel_z, beam_number, pressure, glider_z, measurement_z, glider_w_from_p, ad2cp_dict, amp_binned)
         profiles_dict[index] = profile
     # Add the per profile info to the mission summary
     for extra in extras_list: