From 79c8aa7c680152bcfc0069def61955e959b7b087 Mon Sep 17 00:00:00 2001 From: Md Bayazid Bostame Date: Wed, 9 Oct 2024 10:50:10 +0200 Subject: [PATCH] Session method used in the nextcloud configuration and login to efficiently handling it and avoid performance issue --- syn_nextcloud.py | 101 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/syn_nextcloud.py b/syn_nextcloud.py index 957333d..85d3107 100644 --- a/syn_nextcloud.py +++ b/syn_nextcloud.py @@ -1,19 +1,21 @@ import os import requests import time +import json from requests.auth import HTTPBasicAuth from dotenv import load_dotenv from pathlib import Path +from urllib.parse import unquote # Load environment variables from .env file securely env_path = Path(__file__).parent / 'env' / '.env' load_dotenv(dotenv_path=env_path) # Load Nextcloud credentials and base information from the .env file -NEXTCLOUD_BASE_URL = os.getenv('NEXTCLOUD_BASE_URL').rstrip('/') # Remove trailing slash if any +NEXTCLOUD_BASE_URL = os.getenv('NEXTCLOUD_BASE_URL').rstrip('/') NEXTCLOUD_USERNAME = os.getenv('NEXTCLOUD_USERNAME') NEXTCLOUD_PASSWORD = os.getenv('NEXTCLOUD_PASSWORD') -NEXTCLOUD_DIRECTORY = os.getenv('NEXTCLOUD_DIRECTORY').strip('/') # Ensure no leading/trailing slashes +NEXTCLOUD_DIRECTORY = os.getenv('NEXTCLOUD_DIRECTORY').strip('/') # Load the sync interval from the environment or set a default (in seconds) SYNC_INTERVAL = int(os.getenv('SYNC_INTERVAL', 60)) # Default is 60 seconds @@ -24,11 +26,27 @@ local_folders = { 'onboarded_person': 'onboarded_person' } -def get_nextcloud_files(folder_name): +# Local file tracking for uploads +TRACKING_FILE = 'file_tracking.json' + +def load_tracking_data(): + """Load the file tracking data from the tracking file.""" + if os.path.exists(TRACKING_FILE): + with open(TRACKING_FILE, 'r') as f: + return json.load(f) + return {} + +def save_tracking_data(data): + """Save the file tracking data to the tracking file.""" + with open(TRACKING_FILE, 'w') as f: + json.dump(data, f, indent=4) + +def get_nextcloud_files(session, folder_name): """ - Get a list of files from a specific Nextcloud folder (e.g., 'attachments' or 'onboarded_person'). - + Get a list of files from a specific Nextcloud folder using a persistent session. + Args: + session (requests.Session): The persistent session object. folder_name (str): The subfolder name under the group folder on Nextcloud. Returns: @@ -39,22 +57,20 @@ def get_nextcloud_files(folder_name): print(f"Connecting to Nextcloud folder: {nextcloud_url}") # Make a PROPFIND request to list files in the Nextcloud directory - headers = { - 'Depth': '1', # Depth header to get all files in the directory - } - response = requests.request("PROPFIND", nextcloud_url, auth=HTTPBasicAuth(NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD), headers=headers) + headers = {'Depth': '1'} + response = session.request("PROPFIND", nextcloud_url, headers=headers) if response.status_code != 207: raise Exception(f"Failed to list Nextcloud files in {folder_name}. Status code: {response.status_code}") print(f"Successfully retrieved file list from Nextcloud folder: {folder_name}.") - # Extract filenames from the XML response files = [] for line in response.text.split('\n'): if '' in line: filename = line.split('')[1].split('')[0].split('/')[-1] - if filename: # Avoid empty strings + filename = unquote(filename).strip() + if filename: files.append(filename) print(f"Nextcloud '{folder_name}' contains {len(files)} files: {files}") @@ -64,11 +80,12 @@ def get_nextcloud_files(folder_name): print(f"Error retrieving files from Nextcloud folder {folder_name}: {e}") return [] -def upload_file_to_nextcloud(file_path, filename, folder_name): +def upload_file_to_nextcloud(session, file_path, filename, folder_name): """ - Upload a file to a specific Nextcloud folder (e.g., 'attachments' or 'onboarded_person'). + Upload a file to a specific Nextcloud folder using a persistent session. Args: + session (requests.Session): The persistent session object. file_path (str): The full path to the file to upload. filename (str): The name of the file being uploaded. folder_name (str): The Nextcloud subfolder where the file should be uploaded. @@ -78,43 +95,57 @@ def upload_file_to_nextcloud(file_path, filename, folder_name): print(f"Uploading file: {filename} to Nextcloud folder: {folder_name}...") with open(file_path, 'rb') as f: - response = requests.put(nextcloud_url, data=f, auth=HTTPBasicAuth(NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD)) + response = session.put(nextcloud_url, data=f) if response.status_code not in [200, 201, 204]: # 204 is a valid status code for success raise Exception(f"Failed to upload {filename}. Status code: {response.status_code}") else: print(f"Successfully uploaded {filename} to Nextcloud folder: {folder_name}.") + return True except Exception as e: print(f"Error uploading file {filename} to Nextcloud folder {folder_name}: {e}") + return False -def check_for_new_files(local_folder, nextcloud_files, nextcloud_folder): +def check_for_new_files(session, local_folder, nextcloud_files, nextcloud_folder, tracking_data): """ - Check for new files in the local folder and upload them to Nextcloud. + Check for new files in the local folder and upload them to Nextcloud using a persistent session. Args: + session (requests.Session): The persistent session object. local_folder (str): The local folder to check for new files. nextcloud_files (list): The list of files already present in the Nextcloud folder. nextcloud_folder (str): The corresponding Nextcloud folder to upload new files to. + tracking_data (dict): The dictionary containing tracked files and their modification times. """ try: local_files = os.listdir(local_folder) print(f"Files found in local folder '{local_folder}': {local_files}") for file in local_files: - if file not in nextcloud_files: - file_path = os.path.join(local_folder, file) - print(f"New file detected: {file} (from {local_folder})") - upload_file_to_nextcloud(file_path, file, nextcloud_folder) + file_path = os.path.join(local_folder, file) + file_mtime = os.path.getmtime(file_path) + + # Check if the file is new or modified + if file not in nextcloud_files and (file not in tracking_data or tracking_data[file] != file_mtime): + print(f"New or modified file detected: {file} (from {local_folder})") + if upload_file_to_nextcloud(session, file_path, file, nextcloud_folder): + # Update tracking data after successful upload + tracking_data[file] = file_mtime else: - print(f"File {file} already exists in Nextcloud folder {nextcloud_folder}. Skipping...") + print(f"File {file} already exists in Nextcloud folder {nextcloud_folder} or has not been modified. Skipping...") except Exception as e: print(f"Error checking for new files in {local_folder}: {e}") -def sync_folders(): +def sync_folders(session): """ - Check local folders for new files and upload them to the corresponding Nextcloud folders. + Check local folders for new files and upload them to the corresponding Nextcloud folders using a persistent session. + + Args: + session (requests.Session): The persistent session object. """ + tracking_data = load_tracking_data() + try: for local_folder, nextcloud_folder in local_folders.items(): print(f"Checking local folder: {local_folder} for new files...") @@ -126,10 +157,13 @@ def sync_folders(): continue # Get a list of files in the corresponding Nextcloud folder - nextcloud_files = get_nextcloud_files(nextcloud_folder) + nextcloud_files = get_nextcloud_files(session, nextcloud_folder) # Check for new files in the local folder and upload them - check_for_new_files(local_folder, nextcloud_files, nextcloud_folder) + check_for_new_files(session, local_folder, nextcloud_files, nextcloud_folder, tracking_data) + + # Save updated tracking data + save_tracking_data(tracking_data) except Exception as e: print(f"Error during folder synchronization: {e}") @@ -151,12 +185,17 @@ def start_periodic_sync(): Start the periodic synchronization of local folders to Nextcloud. The synchronization interval is specified in the .env file or defaults to 60 seconds. """ - while True: - print("Starting folder synchronization...") - sync_folders() - print("Synchronization complete.") - countdown_timer(SYNC_INTERVAL) # Wait for the specified sync interval + # Use a persistent session for Nextcloud requests + with requests.Session() as session: + # Set up authentication for the session + session.auth = HTTPBasicAuth(NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD) + + while True: + print("Starting folder synchronization...") + sync_folders(session) + print("Synchronization complete.") + countdown_timer(SYNC_INTERVAL) # Wait for the specified sync interval if __name__ == "__main__": - print("Starting periodic folder synchronization service...") + print("Starting periodic folder synchronization service with a persistent session...") start_periodic_sync()