Merge branch 'salman' into etsasa

The formatting is still ugly

README.md

@@ -2,7 +2,7 @@ RAD Webserver
 ==============
 
 A web service to view quicklook plots of MSL RAD data. Built using [Bokeh](https://bokeh.org/) and the
-[RAD Loader](https://gitlab.physik.uni-kiel.de/mslrad/flight_tools).
+[RAD Loader](https://cau-git.rz.uni-kiel.de/ieap/et/ep/mslrad/flight_tools).
 
 The current version of the app is running at https://solar-orbiter.physik.uni-kiel.de/rad/plotter/.
 
@@ -19,7 +19,7 @@ We're going to install the dependencies into a virtual environment, so that they
 
 ```bash
 # Clone the project
-https://gitlab.physik.uni-kiel.de/rad/RAD-webserver.git
+https://cau-git.rz.uni-kiel.de/ieap/et/ep/mslrad/rad-webserver.git
 cd RAD-webserver
 
 # create and activate a virtual environment

plots/Counters.py

@@ -49,31 +49,87 @@ class CountersTimeprofilePlots_4(PlotCollection):
         show(button)
         return(plot)
 
-    
     def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
+        """
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
+        Load and process counter data within a given datetime range.
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        counters = ft.load_data((np.arange(start_sol,end_sol)),load='counters')
+        Args:
+            start (dt.datetime): Start of the datetime range.
+            end (dt.datetime): End of the datetime range.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the processed counter data.
+        """
+        # Ensure start and end are timezone-naive
+        if start.tzinfo is not None:
+            start = start.replace(tzinfo=None)
+        if end.tzinfo is not None:
+            end = end.replace(tzinfo=None)
+
+        # Step 1: Convert datetime to posix and then to sol
+        start_posix = Mtt.dt_to_posix(start)
+        end_posix = Mtt.dt_to_posix(end)
+        start_sol = int(Mtt.posix_to_sol(start_posix))
+        end_sol = int(Mtt.posix_to_sol(end_posix))
+
+        # Step 2: Load counter data for the sol range
+        counters = ft.load_data(np.arange(start_sol, end_sol + 1), load='counters')
+
+        # Apply the clean mask
         msk = ft.get_clean_mask(counters)
 
-        counters_= pd.DataFrame(counters['l2mc'],columns = [4],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+        # Step 3: Convert data into DataFrames with datetime index
-        accTime_T= pd.DataFrame(counters,columns = ['accTime_T'],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+        counters_ = pd.DataFrame(
+            counters['l2mc'],
+            columns=[4],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+        )
+        accTime_T = pd.DataFrame(
+            counters,
+            columns=['accTime_T'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+        )
 
-        frames = [counters_, accTime_T]
+        # Combine and compute the desired column
+        result = pd.concat([counters_, accTime_T], axis=1)
+        result["L2_[4]"] = result[4] / result['accTime_T']
 
-        result = pd.concat(frames, axis=1)
+        counters_4 = pd.DataFrame(
-        result["L2_[4]"] = result[4]/result['accTime_T']
+            result,
+            columns=["L2_[4]"],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+        )
+        sol = pd.DataFrame(
+            counters['sol'],
+            columns=['sol'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+        )
 
-        #print(result)
+        # Step 4: Trim data to match the exact datetime range
+        data_start = counters_4.index.min()
+        data_end = counters_4.index.max()
 
-        counters_4= pd.DataFrame(result, columns = ["L2_[4]"],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+        start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+        end_trimmed = min(end, data_end)       # End can't be later than data_end
 
-        #print(counters_4)
+        exact_range_mask = (counters_4.index >= start_trimmed) & (counters_4.index <= end_trimmed)
-        sol= pd.DataFrame(counters['l2mc'],columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
 
-        return({'counters_4': counters_4[msk], 'sol': sol[msk]})
+        counters_4_trimmed = counters_4[exact_range_mask]
+        sol_trimmed = sol[exact_range_mask]
+
+        # Ensure trimmed data is not empty
+        if counters_4_trimmed.empty:
+            raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
+
+        # Debugging output
+        #print(f"Counter data range: {data_start} - {data_end}")
+        #print(f"Requested start: {start} - end: {end}")
+        #print(f"Trimmed range start: {counters_4_trimmed.index.min()} - end: {counters_4_trimmed.index.max()}")
+
+        return {
+            'counters_4': counters_4_trimmed,
+            'sol': sol_trimmed
+        }
 
 class CountersTimeprofilePlots_5(PlotCollection):
     """
@@ -95,32 +151,87 @@ class CountersTimeprofilePlots_5(PlotCollection):
         button.js_on_click(CustomJS(args=dict(source=source),code=open(os.path.join(os.path.dirname(__file__),"download.js")).read()))
         show(button)
         return(plot)
-
-    
     def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
+            """
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
+            Load and process counter data within a given datetime range.
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        counters = ft.load_data((np.arange(start_sol,end_sol)),load='counters')
+            Args:
-        msk = ft.get_clean_mask(counters)
+                start (dt.datetime): Start of the datetime range.
+                end (dt.datetime): End of the datetime range.
 
-        counters_= pd.DataFrame(counters['l2mc'],columns = [5],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+            Returns:
-        accTime_T= pd.DataFrame(counters,columns = ['accTime_T'],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+                Dict[str, Any]: A dictionary containing the processed counter data.
+            """
+            # Ensure start and end are timezone-naive
+            if start.tzinfo is not None:
+                start = start.replace(tzinfo=None)
+            if end.tzinfo is not None:
+                end = end.replace(tzinfo=None)
 
-        frames = [counters_, accTime_T]
+            # Step 1: Convert datetime to posix and then to sol
+            start_posix = Mtt.dt_to_posix(start)
+            end_posix = Mtt.dt_to_posix(end)
+            start_sol = int(Mtt.posix_to_sol(start_posix))
+            end_sol = int(Mtt.posix_to_sol(end_posix))
 
-        result = pd.concat(frames, axis=1)
+            # Step 2: Load counter data for the sol range
-        result["L2_[5]"] = result[5]/result['accTime_T']
+            counters = ft.load_data(np.arange(start_sol, end_sol + 1), load='counters')
 
-        #print(result)
+            # Apply the clean mask
+            msk = ft.get_clean_mask(counters)
 
-        counters_5= pd.DataFrame(result, columns = ["L2_[5]"],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+            # Step 3: Convert data into DataFrames with datetime index
+            counters_ = pd.DataFrame(
+                counters['l2mc'],
+                columns=[5],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
+            accTime_T = pd.DataFrame(
+                counters,
+                columns=['accTime_T'],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
 
-        #print(counters_4)
+            # Combine and compute the desired column
-        sol= pd.DataFrame(counters['l2mc'],columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+            result = pd.concat([counters_, accTime_T], axis=1)
+            result["L2_[5]"] = result[5] / result['accTime_T']
 
-        return({'counters_5': counters_5[msk], 'sol': sol[msk]})
+            counters_5 = pd.DataFrame(
+                result,
+                columns=["L2_[5]"],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
+            sol = pd.DataFrame(
+                counters['sol'],
+                columns=['sol'],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
+
+            # Step 4: Trim data to match the exact datetime range
+            data_start = counters_5.index.min()
+            data_end = counters_5.index.max()
+
+            start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+            end_trimmed = min(end, data_end)       # End can't be later than data_end
+
+            exact_range_mask = (counters_5.index >= start_trimmed) & (counters_5.index <= end_trimmed)
+
+            counters_5_trimmed = counters_5[exact_range_mask]
+            sol_trimmed = sol[exact_range_mask]
+
+            # Ensure trimmed data is not empty
+            if counters_5_trimmed.empty:
+                raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
+
+            # Debugging output
+            #print(f"Counter data range: {data_start} - {data_end}")
+            #print(f"Requested start: {start} - end: {end}")
+            #print(f"Trimmed range start: {counters_5_trimmed.index.min()} - end: {counters_5_trimmed.index.max()}")
+
+            return {
+                'counters_5': counters_5_trimmed,
+                'sol': sol_trimmed
+            }
 
 class CountersTimeprofilePlots_15(PlotCollection):
     """
@@ -143,183 +254,80 @@ class CountersTimeprofilePlots_15(PlotCollection):
         show(button)
         return(plot)
 
-    
     def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
+            """
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
+            Load and process counter data within a given datetime range.
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        counters = ft.load_data((np.arange(start_sol,end_sol)),load='counters')
+            Args:
-        msk = ft.get_clean_mask(counters)
+                start (dt.datetime): Start of the datetime range.
+                end (dt.datetime): End of the datetime range.
 
-        counters_= pd.DataFrame(counters['l2mc'],columns = [15],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+            Returns:
-        accTime_T= pd.DataFrame(counters,columns = ['accTime_T'],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+                Dict[str, Any]: A dictionary containing the processed counter data.
+            """
+            # Ensure start and end are timezone-naive
+            if start.tzinfo is not None:
+                start = start.replace(tzinfo=None)
+            if end.tzinfo is not None:
+                end = end.replace(tzinfo=None)
 
-        frames = [counters_, accTime_T]
+            # Step 1: Convert datetime to posix and then to sol
+            start_posix = Mtt.dt_to_posix(start)
+            end_posix = Mtt.dt_to_posix(end)
+            start_sol = int(Mtt.posix_to_sol(start_posix))
+            end_sol = int(Mtt.posix_to_sol(end_posix))
 
-        result = pd.concat(frames, axis=1)
+            # Step 2: Load counter data for the sol range
-        result["L2_[15]"] = result[15]/result['accTime_T']
+            counters = ft.load_data(np.arange(start_sol, end_sol + 1), load='counters')
 
-        #print(result)
+            # Apply the clean mask
+            msk = ft.get_clean_mask(counters)
 
-        counters_15= pd.DataFrame(result, columns = ["L2_[15]"],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+            # Step 3: Convert data into DataFrames with datetime index
+            counters_ = pd.DataFrame(
+                counters['l2mc'],
+                columns=[15],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
+            accTime_T = pd.DataFrame(
+                counters,
+                columns=['accTime_T'],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
 
-        #print(counters_4)
+            # Combine and compute the desired column
-        sol= pd.DataFrame(counters['l2mc'],columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']),name ='date'))
+            result = pd.concat([counters_, accTime_T], axis=1)
+            result["L2_[15]"] = result[15] / result['accTime_T']
 
-        return({'counters_15': counters_15[msk], 'sol': sol[msk]})
+            counters_15 = pd.DataFrame(
+                result,
+                columns=["L2_[15]"],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
+            sol = pd.DataFrame(
+                counters['sol'],
+                columns=['sol'],
+                index=pd.DatetimeIndex(Mtt.sol2datetime(counters['sol']), name='date')
+            )
 
-"""
+            # Step 4: Trim data to match the exact datetime range
-class DoseTimeprofilePlotsE(PlotCollection):
+            data_start = counters_15.index.min()
+            data_end = counters_15.index.max()
 
-    #Plot Total ionising dose
+            start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+            end_trimmed = min(end, data_end)       # End can't be later than data_end
 
-    group = 'Dose-RAD'
+            exact_range_mask = (counters_15.index >= start_trimmed) & (counters_15.index <= end_trimmed)
-    name = 'Dose-E'
 
-    def create_plots(self, data: Dict[str, Any]) -> Dict[str, Plot]:
+            counters_15_trimmed = counters_15[exact_range_mask]
-        plot =  {'dose_E': LinePlot(self, data['dose_E']*8.64*1e7, title='RAD_E measurements', ylabel='dose rate [mGy/day]') }
+            sol_trimmed = sol[exact_range_mask]
-        A_sol = data['sol']['sol']
-        source = ColumnDataSource(({'A_sol':A_sol, 'A_time': ([x.strftime("%Y-%m-%d %H:%M:%S")for x in data['dose_E']['E'].index]),'E_dose [mGy/day]': data['dose_E']['E']*8.64*1e7}))
-        columns = [
-                TableColumn(field='E_dose [mGy/day]', title="E dose rate [mGy/day]"),
-                ]
-
-        table = DataTable(
-                    source=source,
-                    columns=columns,
-                    width=400,
-                    height=600,
-                    sortable=True,
-                    selectable=True,
-                    editable=True,
-                )
-
-        button = Button(label="Download", button_type="success")
-        button.js_on_click(CustomJS(args=dict(source=source),code=open(os.path.join(os.path.dirname(__file__),"download.js")).read()))
-        show(button)
-
-        return(plot)
-    
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
-
-        dose = ft.load_data((np.arange(start_sol,end_sol)),load='dose', force_timesort=True)
-        msk = ft.get_clean_mask(dose)
-
-        dose_E= pd.DataFrame(dose,columns = ['E'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
-        sol= pd.DataFrame(dose,columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
-        
-        return({'dose_E': dose_E[msk], 'sol': sol[msk]})
-
-
-class DoseTimeprofilePlotsB_E(PlotCollection):
-    
-    #Plot Total ionising dose
-    
-    group = 'Dose-RAD'
-    name = 'Dose-BE'
-    def create_plots(self, data: Dict[str, Any]) -> Dict[str, Plot]:
-        plot = {'dose_BE': LinePlot(self, data['dose_BE']*8.64*1e7, title='RAD_BE measurements', ylabel='dose rate [mGy/day]')}
-        A_sol = data['sol']['sol']
-        source = ColumnDataSource(({'A_sol':A_sol, 'A_time': ([x.strftime("%Y-%m-%d %H:%M:%S")for x in data['dose_BE']['B'].index]),'B_dose[mGy/day][per day (i.e., 24 hours), not Sol]': data['dose_BE']['B']*8.64*1e7, 'E_dose[mGy/day][per day (i.e., 24 hours), not Sol]': data['dose_BE']['E']*8.64*1e7}))
-        columns = [
-                TableColumn(field='B_dose [mGy/day]', title="B dose rate [mGy/day]"),
-                TableColumn(field='E_dose [mGy/day]', title="E dose rate [mGy/day]"),
-                ]
-
-        table = DataTable(
-                    source=source,
-                    columns=columns,
-                    width=400,
-                    height=600,
-                    sortable=True,
-                    selectable=True,
-                    editable=True,
-                )
-
-        button = Button(label="Download", button_type="success")
-        button.js_on_click(CustomJS(args=dict(source=source),code=open(os.path.join(os.path.dirname(__file__),"download.js")).read()))
-        show(button)
-        
-
-        return(plot)
-
-    
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
-
-        dose = ft.load_data((np.arange(start_sol,end_sol)),load='dose', force_timesort=True)
-        msk = ft.get_clean_mask(dose, high_dose_resolution=False)
-
-        dose_BE= pd.DataFrame(dose,columns = ['B','E'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol'])))
-        dose_BE.index.name = 'date'
-        #dose_E= pd.DataFrame(dose,columns = ['B'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
-        sol= pd.DataFrame(dose,columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
-
-        return({'dose_BE': dose_BE[msk], 'sol': sol[msk], 'start':start})
-        
-    
-class Test_plot(PlotCollection):
-    name = 'Test Plot'
-    group = 'Test'
-
-    def create_plots(self, data: Dict[str, Any]) -> Dict[str, Plot]:
-        # create two panels, each showing a line plot
-        return {
-            'foo': LinePlot(self, data['foo'],
-                            title='Foo measurements',
-                            ylabel='Temperature [°C]'),
-            #'bar': LinePlot(self, data['bar'],
-             #               title='Bar measurements',
-              #              ylabel='Voltage [V]'),
-        }
-
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        # just generate some example data for the plots:
-        return {
-            'foo': pd.DataFrame({
-                #'a': [2, 2],
-                'b': [2, 1]
-            }, index=pd.DatetimeIndex([start, end], name='date'))
-            #, 'bar': pd.DataFrame({
-             #   'x': [100, 222],
-              #  'y': [111, 150]
-            #}, index=pd.DatetimeIndex([start, end], name='date'))
-        }
-        
-  """
-  
-        
-  #       savebutton = Button(label="Save", button_type="success")
-  #       callback = CustomJS(
-  #           args=dict(source_data=source),
-  #           code="""
- #               var inds = source_data.selected.indices;
- #               var data = source_data.data;
- #               var out = "x, y, var inds, var data\\n";
- #               for (var i = 0; i < inds.length; i++) {
- #                   out += data['X1'][inds[i]] + "," + data['Y1'][inds[i]] + "," + data['Y2'][inds[i]] +"\\n";
- #               }
- #               var file = new Blob([out], {type: 'text/plain'});
- #               var elem = window.document.createElement('a');
- #               elem.href = window.URL.createObjectURL(file);
- #               elem.download = 'selected-data.txt';
- #               document.body.appendChild(elem);
- #               elem.click();
- #               document.body.removeChild(elem);
- #               """,
- #       )
- #       savebutton.js_on_click(callback)
-
- #       layout = grid([table, savebutton], ncols=2)
- #       show(layout)
- #                   out += data['X1'][inds[i]] + "," + data['Y1'][inds[i]] + "," + data['Y2'][inds[i]] +"\\n";
-  #      
 
+            # Ensure trimmed data is not empty
+            if counters_15_trimmed.empty:
+                raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
 
+            return {
+                'counters_15': counters_15_trimmed,
+                'sol': sol_trimmed
+            }
 

plots/Dose.py

@@ -49,7 +49,85 @@ class DoseTimeprofilePlotsB(PlotCollection):
         show(button)
         return(plot)
 
+    def load_data(self, start: dt.datetime, end: dt.datetime):
+        """
+        Load data within a datetime range, ensuring the result matches the exact input range.
 
+        Args:
+            start (dt.datetime): Start of the datetime range.
+            end (dt.datetime): End of the datetime range.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the filtered data.
+        """
+        # Ensure start and end are timezone-naive
+        if start.tzinfo is not None:
+            start = start.replace(tzinfo=None)
+        if end.tzinfo is not None:
+            end = end.replace(tzinfo=None)
+
+        # Step 1: Convert datetime to posix time
+        start_posix = Mtt.dt_to_posix(start)
+        end_posix = Mtt.dt_to_posix(end)
+
+        # Step 2: Convert to sol and load data
+        start_sol = int(Mtt.posix_to_sol(start_posix))
+        end_sol = int(Mtt.posix_to_sol(end_posix))
+
+        # Load data for the range
+        dose = ft.load_data(np.arange(start_sol, end_sol + 1), load='dose')
+
+        if not dose:
+            raise ValueError("No data returned for the given range.")
+
+        # Apply the clean mask
+        msk = ft.get_clean_mask(dose)
+
+        # Convert to DataFrame
+        dose_B = pd.DataFrame(
+            dose,
+            columns=['B'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']), name='date')
+        )
+        sol = pd.DataFrame(
+            dose,
+            columns=['sol'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']), name='date')
+        )
+
+        # Ensure dose_B.index is timezone-naive
+        dose_B.index = dose_B.index.tz_localize(None)
+
+        # Step 3: Adjust trimming logic
+        # Instead of strict trimming, rely on the actual data bounds
+        data_start = dose_B.index.min()
+        data_end = dose_B.index.max()
+
+        # Ensure that start and end are within data bounds
+        start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+        end_trimmed = min(end, data_end)       # End can't be later than data_end
+
+        # Create a final mask based on actual data bounds
+        exact_range_mask = (dose_B.index >= start_trimmed) & (dose_B.index <= end_trimmed)
+
+        dose_B_trimmed = dose_B[exact_range_mask]
+        sol_trimmed = sol[exact_range_mask]
+
+        # Ensure trimmed data is not empty
+        if dose_B_trimmed.empty:
+            raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
+
+        # Debugging output
+        #print(f"RAD data range: {data_start} - {data_end}")
+        #print(f"Requested start: {start} - end: {end}")
+        #print(f"Trimmed range start: {dose_B_trimmed.index.min()} - end: {dose_B_trimmed.index.max()}")
+
+        return {
+            'dose_B': dose_B_trimmed,
+            'sol': sol_trimmed
+        }
+    
+    """
     def load_data(self, start: dt.datetime, end: dt.datetime):
         #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
         start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
@@ -60,9 +138,8 @@ class DoseTimeprofilePlotsB(PlotCollection):
 
         dose_B= pd.DataFrame(dose,columns = ['B'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
         sol= pd.DataFrame(dose,columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
-       
         return({'dose_B': dose_B[msk], 'sol': sol[msk]})
-
+    """
 class DoseTimeprofilePlotsE(PlotCollection):
     """
     Plot Total ionising dose
@@ -95,17 +172,82 @@ class DoseTimeprofilePlotsE(PlotCollection):
         return(plot)
  
     def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
+        """
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
+        Load data within a datetime range, ensuring the result matches the exact input range.
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        dose = ft.load_data((np.arange(start_sol,end_sol)),load='dose', force_timesort=True)
+        Args:
+            start (dt.datetime): Start of the datetime range.
+            end (dt.datetime): End of the datetime range.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the filtered data.
+        """
+        # Ensure start and end are timezone-naive
+        if start.tzinfo is not None:
+            start = start.replace(tzinfo=None)
+        if end.tzinfo is not None:
+            end = end.replace(tzinfo=None)
+
+        # Step 1: Convert datetime to posix time
+        start_posix = Mtt.dt_to_posix(start)
+        end_posix = Mtt.dt_to_posix(end)
+
+        # Step 2: Convert to sol and load data
+        start_sol = int(Mtt.posix_to_sol(start_posix))
+        end_sol = int(Mtt.posix_to_sol(end_posix))
+
+        # Load data for the range
+        dose = ft.load_data(np.arange(start_sol, end_sol + 1), load='dose')
+
+        if not dose:
+            raise ValueError("No data returned for the given range.")
+
+        # Apply the clean mask
         msk = ft.get_clean_mask(dose)
 
-        dose_E= pd.DataFrame(dose,columns = ['E'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
+        # Convert to DataFrame
-        sol= pd.DataFrame(dose,columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
+        dose_E = pd.DataFrame(
+            dose,
+            columns=['E'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']), name='date')
+        )
+        sol = pd.DataFrame(
+            dose,
+            columns=['sol'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']), name='date')
+        )
 
-        return({'dose_E': dose_E[msk], 'sol': sol[msk]})
+        # Ensure dose_B.index is timezone-naive
+        dose_E.index = dose_E.index.tz_localize(None)
+
+        # Step 3: Adjust trimming logic
+        # Instead of strict trimming, rely on the actual data bounds
+        data_start = dose_E.index.min()
+        data_end = dose_E.index.max()
+
+        # Ensure that start and end are within data bounds
+        start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+        end_trimmed = min(end, data_end)       # End can't be later than data_end
+
+        # Create a final mask based on actual data bounds
+        exact_range_mask = (dose_E.index >= start_trimmed) & (dose_E.index <= end_trimmed)
+
+        dose_E_trimmed = dose_E[exact_range_mask]
+        sol_trimmed = sol[exact_range_mask]
+
+        # Ensure trimmed data is not empty
+        if dose_E_trimmed.empty:
+            raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
+
+        # Debugging output
+        #print(f"RAD data range: {data_start} - {data_end}")
+        #print(f"Requested start: {start} - end: {end}")
+        #print(f"Trimmed range start: {dose_E_trimmed.index.min()} - end: {dose_E_trimmed.index.max()}")
+
+        return {
+            'dose_E': dose_E_trimmed,
+            'sol': sol_trimmed
+        }
 
 
 class DoseTimeprofilePlotsB_E(PlotCollection):
@@ -136,79 +278,72 @@ class DoseTimeprofilePlotsB_E(PlotCollection):
         button = Button(label="Download", button_type="success")
         button.js_on_click(CustomJS(args=dict(source=source),code=open(os.path.join(os.path.dirname(__file__),"download.js")).read()))
         show(button)        
-        
-
         return(plot)
 
-    
     def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
+        # Ensure start and end are timezone-naive
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
+        if start.tzinfo is not None:
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
+            start = start.replace(tzinfo=None)
+        if end.tzinfo is not None:
+            end = end.replace(tzinfo=None)
 
-        dose = ft.load_data((np.arange(start_sol,end_sol)),load='dose', force_timesort=True)
+        # Step 1: Convert datetime to posix time
-        msk = ft.get_clean_mask(dose, high_dose_resolution=False)
+        start_posix = Mtt.dt_to_posix(start)
+        end_posix = Mtt.dt_to_posix(end)
 
-        dose_BE= pd.DataFrame(dose,columns = ['B','E'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol'])))
+        # Step 2: Convert to sol and load data
-        dose_BE.index.name = 'date'
+        start_sol = int(Mtt.posix_to_sol(start_posix))
-        #dose_E= pd.DataFrame(dose,columns = ['B'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
+        end_sol = int(Mtt.posix_to_sol(end_posix))
-        sol= pd.DataFrame(dose,columns = ['sol'],index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']),name ='date'))
 
-        return({'dose_BE': dose_BE[msk], 'sol': sol[msk], 'start':start})
+        # Load data for the range
+        dose = ft.load_data(np.arange(start_sol, end_sol + 1), load='dose')
 
+        if not dose:
+            raise ValueError("No data returned for the given range.")
 
-class Test_plot(PlotCollection):
+        # Apply the clean mask
-    name = 'Test Plot'
+        msk = ft.get_clean_mask(dose)
-    group = 'Test'
+
+        # Convert to DataFrame
+        dose_BE = pd.DataFrame(
+            dose,
+            columns=['B','E'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']), name='date')
+        )
+        sol = pd.DataFrame(
+            dose,
+            columns=['sol'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(dose['sol']), name='date')
+        )
+
+        # Ensure dose_B.index is timezone-naive
+        dose_BE.index = dose_BE.index.tz_localize(None)
+
+        # Step 3: Adjust trimming logic
+        # Instead of strict trimming, rely on the actual data bounds
+        data_start = dose_BE.index.min()
+        data_end = dose_BE.index.max()
+
+        # Ensure that start and end are within data bounds
+        start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+        end_trimmed = min(end, data_end)       # End can't be later than data_end
+
+        # Create a final mask based on actual data bounds
+        exact_range_mask = (dose_BE.index >= start_trimmed) & (dose_BE.index <= end_trimmed)
+
+        dose_BE_trimmed = dose_BE[exact_range_mask]
+        sol_trimmed = sol[exact_range_mask]
+
+        # Ensure trimmed data is not empty
+        if dose_BE_trimmed.empty:
+            raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
+
+        # Debugging output
+        #print(f"RAD data range: {data_start} - {data_end}")
+        #print(f"Requested start: {start} - end: {end}")
+        #print(f"Trimmed range start: {dose_BE_trimmed.index.min()} - end: {dose_BE_trimmed.index.max()}")
 
-    def create_plots(self, data: Dict[str, Any]) -> Dict[str, Plot]:
-        # create two panels, each showing a line plot
         return {
-            'foo': LinePlot(self, data['foo'],
+            'dose_BE': dose_BE_trimmed,
-                            title='Foo measurements',
+            'sol': sol_trimmed
-                            ylabel='Temperature [°C]'),
-            #'bar': LinePlot(self, data['bar'],
-             #               title='Bar measurements',
-              #              ylabel='Voltage [V]'),
         }
-
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        # just generate some example data for the plots:
-        return {
-            'foo': pd.DataFrame({
-                #'a': [2, 2],
-                'b': [2, 1]
-            }, index=pd.DatetimeIndex([start, end], name='date'))
-            #, 'bar': pd.DataFrame({
-             #   'x': [100, 222],
-              #  'y': [111, 150]
-            #}, index=pd.DatetimeIndex([start, end], name='date'))
-        }
-        
-    
-        """
-        savebutton = Button(label="Save", button_type="success")
-        callback = CustomJS(
-            args=dict(source_data=source),
-            code="""
- #               var inds = source_data.selected.indices;
- #               var data = source_data.data;
- #               var out = "x, y, var inds, var data\\n";
- #               for (var i = 0; i < inds.length; i++) {
- #                   out += data['X1'][inds[i]] + "," + data['Y1'][inds[i]] + "," + data['Y2'][inds[i]] +"\\n";
- #               }
- #               var file = new Blob([out], {type: 'text/plain'});
- #               var elem = window.document.createElement('a');
- #               elem.href = window.URL.createObjectURL(file);
- #               elem.download = 'selected-data.txt';
- #               document.body.appendChild(elem);
- #               elem.click();
- #               document.body.removeChild(elem);
- #               """,
- #       )
- #       savebutton.js_on_click(callback)
-
- #       layout = grid([table, savebutton], ncols=2)
- #       show(layout)
- #                   out += data['X1'][inds[i]] + "," + data['Y1'][inds[i]] + "," + data['Y2'][inds[i]] +"\\n";
-  #      """

plots/LET.py

@@ -63,10 +63,7 @@ class LETPlots(PlotCollection):
 
         #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
         start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
-        print('NOW here')
         end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
-
-        print('I am here', start_sol, end_sol)
         #let=lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True)
 
         return{'Let': pd.DataFrame(lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True),columns = ['X1','Y1','X2','Y2'])}
@@ -87,7 +84,6 @@ class LETPlots_A1B(PlotCollection):
         start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
         end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        print('I am here', start_sol, end_sol)
         let=lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True)
 
         let_A1= pd.DataFrame(let,columns = ['X1','Y1'])
@@ -109,112 +105,8 @@ class LETPlots_A2B(PlotCollection):
         start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
         end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        print('I am here', start_sol, end_sol)
         let=lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True)
 
         let_A2= pd.DataFrame(let,columns = ['X2','Y2'])
         
         return({'Let_A2':let_A2 })
-    
-'''
-    group = 'LET'
-    name = 'Let_A1B'
-    def create_plots(self, data: Dict[str, Any]) -> Dict[str, Plot]:
-        #plot =  {'Dose-B': plt.plot(data['E'])}
-        #temp = pd.DataFrame(data['Frontal'])
-        #print(data['data'])
-        #print(data['Y1'])
-
-        return {'LET_A1B': AccumulatedSpectrumPlot(self, data, title='Total Dose', logy=True,ylabel=r"Total Ionizing Dose[μGy hour⁻¹]")}
-
-
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
-
-        print('I am here', start_sol, end_sol)
-        let=lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True)
-        #let_tot = pd.DataFrame(let, columns = ['X1','X1_err','Y1','Y1_err','X2','X2_err','Y2','Y2_err'] )
-        #print(let_tot['X1'])
-        index = pd.Index(let['X1'])
-
-        let_A1= pd.DataFrame(let,columns = ['Y1'], index = index)
-        
-        let_err =pd.DataFrame(let,columns = ['Y1_err'], index = index)
-
-        #print('Hello',let_A1)
-        
-
-        let_  = pd.concat([ let_A1, let_err], axis = 1, keys=['data','yerr'])
-        print(let_)
-        return({'LET_A1B': let_ })
-    """
-
-
-
-    group = 'LET'
-    name = 'Let_A2B'
-    def create_plots(self, data):
-        #plot =  {'Dose-B': plt.plot(data['E'])}
-        #temp = pd.DataFrame(data['Frontal'])
-        #print(data['X1'])
-        #print(data['Y1'])
-
-        return {'Let': AccumulatedSpectrumPlot(self, data, title='Total Dose', logy=True,ylabel=r"Total Ionizing Dose[μGy hour⁻¹]")}
-
-
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
-
-        print('I am here', start_sol, end_sol)
-        let=lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True)
-
-        let_A1= pd.DataFrame(let,columns = ['X1','Y1'])
-
-        print('Hello',let_A1)
-        
-
-        #let_  = pd.DataFrame(let, keys=['X1','X1_err','Y1','Y1_err','X2','X2_err','Y2','Y2_err'])
-        
-        return({'X1': let_A1,'Y1': let_A1 })
-
-
-
-
-
-    
-    group = 'LET'
-    name = 'Let_A1*B'
-    def create_plots(self, data):
-        #plot =  {'Dose-B': plt.plot(data['E'])}
-        #temp = pd.DataFrame(data['Frontal'])
-        #print(data['X1'])
-        #print(data['Y1'])
-
-        return {'Let': AccumulatedSpectrumPlot(self, data, title='Total Dose', logy=True,ylabel=r"Total Ionizing Dose[μGy hour⁻¹]")}
-
-
-    def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
-
-        print('I am here', start_sol, end_sol)
-        let=lt.LET_hist(np.arange(start_sol,end_sol),  day='sol', scaled=True, N_prio=[1, 2, 3], PHA=True, obs_average=True, bin_num=None, bin_start=50.0, bin_end=100000.0)
-
-        let_A1= pd.DataFrame(let,columns = ['X1','Y1','X2','Y2'])
-        #let_A2= pd.DataFrame(let,columns = ['X2','Y2'])
-
-
-        print('Hello',let_A1)
-        #print('Hello',let_A2)
-
-        
-
-        #let_  = pd.DataFrame(let, keys=['X1','X1_err','Y1','Y1_err','X2','X2_err','Y2','Y2_err'])
-        
-        return({'X1': let_A1,'Y1': let_A1,'X2': let_A1,'Y2': let_A1 })
-    '''

plots/Temp.py

@@ -5,6 +5,7 @@ import numpy as np
 import pandas as pd
 import scipy.constants as const
 import MSL.time_tools  as Mtt
+import MSL.flight_reader as fr
 
 import MSL.flight_tools as ft
 import MSL.LET_tools as lt
@@ -20,17 +21,78 @@ class TempTimeprofilePlots(PlotCollection):
         return {'Temp': LinePlot(self, data['Temp'], title='RAD_Temp measurements', ylabel='Temperature(°C)') }
 
     def load_data(self, start: dt.datetime, end: dt.datetime):
-        #dose  = LNDData(start, end).xmas.dose.to_frame()*1e6
+        """
-        start_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(start)))
+        Load data within a datetime range, ensuring the result matches the exact input range.
-        end_sol = int(Mtt.posix_to_sol(Mtt.dt_to_posix(end)))
 
-        print('I am here', start_sol, end_sol)
+        Args:
-        temp = ft.load_data((np.arange(start_sol,end_sol)),day = 'sol',load='temp')
+            start (dt.datetime): Start of the datetime range.
+            end (dt.datetime): End of the datetime range.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the filtered data.
+        """
+        # Ensure start and end are timezone-naive
+        if start.tzinfo is not None:
+            start = start.replace(tzinfo=None)
+        if end.tzinfo is not None:
+            end = end.replace(tzinfo=None)
+
+        # Step 1: Convert datetime to posix time
+        start_posix = Mtt.dt_to_posix(start)
+        end_posix = Mtt.dt_to_posix(end)
+
+        # Step 2: Convert to sol and load data
+        start_sol = int(Mtt.posix_to_sol(start_posix))
+        end_sol = int(Mtt.posix_to_sol(end_posix))
+
+        # Load data for the range
+        temp = ft.load_data(np.arange(start_sol, end_sol + 1), load='temp')
+
+        if not temp:
+            raise ValueError("No data returned for the given range.")
+
+        # Apply the clean mask
         msk = ft.get_clean_mask(temp)
 
-        print("Now here")
+        # Convert to DataFrame
-        Temp= pd.DataFrame(temp,columns = ['temp'],index=pd.DatetimeIndex(Mtt.sol2datetime(temp['sol']),name ='date'))
+        Temp = pd.DataFrame(
+            temp,
+            columns=['temp'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(temp['sol']), name='date')
+        )
+        sol = pd.DataFrame(
+            temp,
+            columns=['sol'],
+            index=pd.DatetimeIndex(Mtt.sol2datetime(temp['sol']), name='date')
+        )
 
+        # Ensure dose_B.index is timezone-naive
+        Temp.index = Temp.index.tz_localize(None)
 
-        return({'Temp': Temp[msk]})
+        # Step 3: Adjust trimming logic
+        # Instead of strict trimming, rely on the actual data bounds
+        data_start = Temp.index.min()
+        data_end = Temp.index.max()
 
+        # Ensure that start and end are within data bounds
+        start_trimmed = max(start, data_start)  # Start can't be earlier than data_start
+        end_trimmed = min(end, data_end)       # End can't be later than data_end
+
+        # Create a final mask based on actual data bounds
+        exact_range_mask = (Temp.index >= start_trimmed) & (Temp.index <= end_trimmed)
+
+        Temp_trimmed = Temp[exact_range_mask]
+        sol_trimmed = sol[exact_range_mask]
+
+        # Ensure trimmed data is not empty
+        if Temp_trimmed.empty:
+            raise ValueError("No data in the trimmed range. Verify the inputs and data availability.")
+        # Debugging output
+        #print(f"RAD data range: {data_start} - {data_end}")
+        #print(f"Requested start: {start} - end: {end}")
+        #print(f"Trimmed range start: {Temp_trimmed.index.min()} - end: {Temp_trimmed.index.max()}")
+
+        return {
+            'Temp': Temp_trimmed,
+            'sol': sol_trimmed
+        }

templates/base.html

@@ -12,7 +12,7 @@
         <img class="logo" src="_static/RAD.jpg"/>
         <h1 class="title">MSL RAD data plotter</h1>
         <div class="corner-ribbon">
-            <a href="https://gitlab.physik.uni-kiel.de/rad/rad-webserver">Contribute on GitLab</a>
+            <a href="https://cau-git.rz.uni-kiel.de/ieap/et/ep/mslrad/rad-webserver">Contribute on GitLab</a>
         </div>
     </div>
     <div class="disclaimer">