From d87d5896325639112e9473d392a169d438588e42 Mon Sep 17 00:00:00 2001 From: Tarun Prajapati Date: Sun, 17 Mar 2024 22:04:09 +0530 Subject: [PATCH 1/4] Script to fetch OpenMEteo Data(NWP Forecast and Historical data) --- OpenMeteo/fetch_OpenMeteo_data.py | 114 ++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 OpenMeteo/fetch_OpenMeteo_data.py diff --git a/OpenMeteo/fetch_OpenMeteo_data.py b/OpenMeteo/fetch_OpenMeteo_data.py new file mode 100644 index 00000000..34492066 --- /dev/null +++ b/OpenMeteo/fetch_OpenMeteo_data.py @@ -0,0 +1,114 @@ +import openmeteo_requests # Importing required libraries +import requests_cache +import pandas as pd +from retry_requests import retry + +class WeatherDataFetcher: + BASE_URL = "https://api.open-meteo.com/v1/" # Base URL for OpenMeteo API + + def __init__(self): + # Initialize the WeatherDataFetcher class + # Setup the Open-Meteo API client with cache and retry on error + cache_session = requests_cache.CachedSession('.cache', expire_after=3600) + retry_session = retry(cache_session, retries=5, backoff_factor=0.2) + self.openmeteo = openmeteo_requests.Client(session=retry_session) + + def fetch_forecast_data(self, NWP, params): + # Fetch weather data from OpenMeteo API for the specified model (NWP) and parameters + url = f"https://api.open-meteo.com/v1/{NWP}" # Construct API URL + try: + responses = self.openmeteo.weather_api(url, params=params) # Get weather data + return responses[0] # Return the first response (assuming only one location) + except openmeteo_requests.OpenMeteoRequestsError as e: + # Handle OpenMeteoRequestsError exceptions + if 'No data is available for this location' in str(e): + print(f"Error: No data available for the location for model '{NWP}'.") + else: + print(f"Error: {e}") + return None + + def fetch_historical_data(self, params): + # Fetch historical weather data from OpenMeteo API + BASE_URL = "https://archive-api.open-meteo.com/v1/archive" + try: + responses = self.openmeteo.weather_api(BASE_URL, params=params) + return responses[0] if responses else None + except ValueError as e: + print(f"Error: {e}") + return None + + def process_hourly_data(self, response): + # Process hourly data from OpenMeteo API response + # Extract hourly data from the response + hourly = response.Hourly() + + # Extract variables + hourly_variables = { + "temperature_2m": hourly.Variables(0).ValuesAsNumpy(), + "relative_humidity_2m": hourly.Variables(1).ValuesAsNumpy(), + "precipitation": hourly.Variables(2).ValuesAsNumpy(), + "cloud_cover": hourly.Variables(3).ValuesAsNumpy() + } + + # Extract time information + time_range = pd.date_range( + start=pd.to_datetime(hourly.Time(), unit="s", utc=True), + end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True), + freq=pd.Timedelta(seconds=hourly.Interval()), + inclusive="left" + ) + + # Create a dictionary for hourly data + hourly_data = {"date": time_range} + + # Assign each variable to the corresponding key in the dictionary + for variable_name, variable_values in hourly_variables.items(): + hourly_data[variable_name] = variable_values + + # Create a DataFrame from the dictionary + hourly_dataframe = pd.DataFrame(data=hourly_data) + return hourly_dataframe + + def print_location_info(self, response): + # Print location information from OpenMeteo API response + print(f"Coordinates {response.Latitude()}°N {response.Longitude()}°E") + print(f"Elevation {response.Elevation()} m asl") + print(f"Timezone {response.Timezone()} {response.TimezoneAbbreviation()}") + print(f"Timezone difference to GMT+0 {response.UtcOffsetSeconds()} s") + + +def main(): + # Main function to demonstrate usage of WeatherDataFetcher class + fetcher = WeatherDataFetcher() # Create instance of WeatherDataFetcher + + # Specify parameters for weather data fetch + NWP = "gfs" # Choose NWP model + + # NWP models = ["dwd-icon", "gfs", "ecmwf", "meteofrance", "jma", "metno", "gem", "bom", "cma"] + + params = { + "latitude": 40.77, # Latitude of the location + "longitude": -73.91, # Longitude of the location + "hourly": ["temperature_2m", "relative_humidity_2m", "precipitation", "cloud_cover"], # Variables to fetch + "start_date": "2023-12-21", # Start date for data + "end_date": "2024-03-15" # End date for data + } + + # Fetch weather data for specified model and parameters + response = fetcher.fetch_forecast_data(NWP, params) + + # Print location information + fetcher.print_location_info(response) + + # Process and print hourly data + gfs_dataframe = fetcher.process_hourly_data(response) + print(gfs_dataframe) + + # Fetch historical weather data + history = fetcher.fetch_historical_data(params) + history_dataframe = fetcher.process_hourly_data(history) + print(history_dataframe) + + +if __name__ == "__main__": + main() # Call main function if script is executed directly From be4a232a75696522f74f338764cef6d747dba84b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 17 Mar 2024 16:51:01 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- OpenMeteo/fetch_OpenMeteo_data.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/OpenMeteo/fetch_OpenMeteo_data.py b/OpenMeteo/fetch_OpenMeteo_data.py index 34492066..a7ba7237 100644 --- a/OpenMeteo/fetch_OpenMeteo_data.py +++ b/OpenMeteo/fetch_OpenMeteo_data.py @@ -1,15 +1,16 @@ import openmeteo_requests # Importing required libraries -import requests_cache import pandas as pd +import requests_cache from retry_requests import retry + class WeatherDataFetcher: BASE_URL = "https://api.open-meteo.com/v1/" # Base URL for OpenMeteo API def __init__(self): # Initialize the WeatherDataFetcher class # Setup the Open-Meteo API client with cache and retry on error - cache_session = requests_cache.CachedSession('.cache', expire_after=3600) + cache_session = requests_cache.CachedSession(".cache", expire_after=3600) retry_session = retry(cache_session, retries=5, backoff_factor=0.2) self.openmeteo = openmeteo_requests.Client(session=retry_session) @@ -21,12 +22,12 @@ def fetch_forecast_data(self, NWP, params): return responses[0] # Return the first response (assuming only one location) except openmeteo_requests.OpenMeteoRequestsError as e: # Handle OpenMeteoRequestsError exceptions - if 'No data is available for this location' in str(e): + if "No data is available for this location" in str(e): print(f"Error: No data available for the location for model '{NWP}'.") else: print(f"Error: {e}") return None - + def fetch_historical_data(self, params): # Fetch historical weather data from OpenMeteo API BASE_URL = "https://archive-api.open-meteo.com/v1/archive" @@ -47,7 +48,7 @@ def process_hourly_data(self, response): "temperature_2m": hourly.Variables(0).ValuesAsNumpy(), "relative_humidity_2m": hourly.Variables(1).ValuesAsNumpy(), "precipitation": hourly.Variables(2).ValuesAsNumpy(), - "cloud_cover": hourly.Variables(3).ValuesAsNumpy() + "cloud_cover": hourly.Variables(3).ValuesAsNumpy(), } # Extract time information @@ -55,7 +56,7 @@ def process_hourly_data(self, response): start=pd.to_datetime(hourly.Time(), unit="s", utc=True), end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True), freq=pd.Timedelta(seconds=hourly.Interval()), - inclusive="left" + inclusive="left", ) # Create a dictionary for hourly data @@ -83,15 +84,20 @@ def main(): # Specify parameters for weather data fetch NWP = "gfs" # Choose NWP model - + # NWP models = ["dwd-icon", "gfs", "ecmwf", "meteofrance", "jma", "metno", "gem", "bom", "cma"] params = { "latitude": 40.77, # Latitude of the location "longitude": -73.91, # Longitude of the location - "hourly": ["temperature_2m", "relative_humidity_2m", "precipitation", "cloud_cover"], # Variables to fetch + "hourly": [ + "temperature_2m", + "relative_humidity_2m", + "precipitation", + "cloud_cover", + ], # Variables to fetch "start_date": "2023-12-21", # Start date for data - "end_date": "2024-03-15" # End date for data + "end_date": "2024-03-15", # End date for data } # Fetch weather data for specified model and parameters From 3bb5cf0a7810cd7b2e8870b196d523d482969401 Mon Sep 17 00:00:00 2001 From: Tarun Prajapati Date: Wed, 20 Mar 2024 16:20:00 +0530 Subject: [PATCH 3/4] OpenMeteo to xarray dataset --- OpenMeteo/OpenMeteo_xr | 82 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 OpenMeteo/OpenMeteo_xr diff --git a/OpenMeteo/OpenMeteo_xr b/OpenMeteo/OpenMeteo_xr new file mode 100644 index 00000000..eb76ace3 --- /dev/null +++ b/OpenMeteo/OpenMeteo_xr @@ -0,0 +1,82 @@ +import openmeteo_requests +import requests_cache +import pandas as pd +from retry_requests import retry +import numpy as np +import xarray as xr +from typing import Tuple, List + +class WeatherDataFetcher: + def __init__(self): + # Setup the Open-Meteo API client with cache and retry on error + cache_session = requests_cache.CachedSession('.cache', expire_after=3600) + retry_session = retry(cache_session, retries=5, backoff_factor=0.2) + self.openmeteo = openmeteo_requests.Client(session=retry_session) + + def generate_lat_lon_grid(self, lat_range: Tuple[float, float] = (-90, 90), lon_range: Tuple[float, float] = (-180, 180), lat_step: float = 0.25, lon_step: float = 0.25) -> Tuple[np.ndarray, np.ndarray]: + latitudes = np.arange(lat_range[0], lat_range[1] + lat_step, lat_step) + longitudes = np.arange(lon_range[0], lon_range[1] + lon_step, lon_step) + return latitudes, longitudes + + def fetch_world_grid_data(self, start_date: str, end_date: str, weather_variables: List[str]) -> xr.Dataset: + # Generate latitude and longitude grid + latitudes, longitudes = self.generate_lat_lon_grid() + + # Split the grid into smaller chunks (adjust as needed) + chunk_size = 200 + latitude_chunks = [latitudes[i:i+chunk_size] for i in range(0, len(latitudes), chunk_size)] + longitude_chunks = [longitudes[i:i+chunk_size] for i in range(0, len(longitudes), chunk_size)] + + all_data = [] + lat = [] + lon = [] + # Make API requests for each chunk of latitude and longitude values + for lat_chunk, lon_chunk in zip(latitude_chunks, longitude_chunks): + params = { + "latitude": lat_chunk.tolist(), + "longitude": lon_chunk.tolist(), + "hourly": weather_variables, + "start_date": start_date, + "end_date": end_date + } + try: + responses = self.openmeteo.weather_api(url, params=params) + + except: + break + res = [lat for lat in lat_chunk.tolist()] + lat+=res + res = [lon for lon in lat_chunk.tolist()] + lon+=res + # Process responses as needed + for response in responses: + data = { + "latitude": response.Latitude(), + "longitude": response.Longitude(), + "date": pd.date_range( + start=pd.to_datetime(response.Hourly().Time(), unit="s", utc=True), + end=pd.to_datetime(response.Hourly().TimeEnd(), unit="s", utc=True), + freq=pd.Timedelta(seconds=response.Hourly().Interval()), + inclusive="left" + ) + } + for var in weather_variables: + data[var] = response.Hourly().Variables(weather_variables.index(var)).ValuesAsNumpy() + + all_data.append(data) + print(len(lat)) + print(all_data[0]["visibility"]) + # Create an xarray dataset from the collected data + dataset = xr.Dataset( + {var: (["latitude", "longitude", "date"], np.array(all_data[i][var])) for i,var in zip(range(len(all_data)),weather_variables)}, + coords={"latitude": lat, "longitude": lon, "date": np.array(all_data[i]["date"] for i in range((len(all_data))))} + ) + return dataset + +# Example usage: +fetcher = WeatherDataFetcher() +start_date = "2024-01-01" +end_date = "2024-01-10" +weather_variables = ["temperature_2m", "precipitation", "visibility", "cloud_cover"] +world_grid_data = fetcher.fetch_world_grid_data(start_date, end_date, weather_variables) +print(world_grid_data) From 2494c84af07eeb379b1ff633e52a835e2855e8d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:52:28 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- OpenMeteo/OpenMeteo_xr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenMeteo/OpenMeteo_xr b/OpenMeteo/OpenMeteo_xr index eb76ace3..66227798 100644 --- a/OpenMeteo/OpenMeteo_xr +++ b/OpenMeteo/OpenMeteo_xr @@ -39,9 +39,9 @@ class WeatherDataFetcher: "start_date": start_date, "end_date": end_date } - try: + try: responses = self.openmeteo.weather_api(url, params=params) - + except: break res = [lat for lat in lat_chunk.tolist()]