ente/onboarding_system
1
0
Fork 0
forked from bostame/onboarding_system
Code Pull Requests Activity

Session method used in the nextcloud configuration and login to efficiently handling it and avoid performance issue

Browse Source
This commit is contained in:
Md Bayazid Bostame
2024-10-09 10:50:10 +02:00
parent bd5ccc0d16
commit 79c8aa7c68
1 changed files with 70 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
syn_nextcloud.py
View File

@@ -1,19 +1,21 @@
import os
import requests
import time
import json
from requests.auth import HTTPBasicAuth
from dotenv import load_dotenv
from pathlib import Path
from urllib.parse import unquote
# Load environment variables from .env file securely
env_path = Path(__file__).parent / 'env' / '.env'
load_dotenv(dotenv_path=env_path)
# Load Nextcloud credentials and base information from the .env file
NEXTCLOUD_BASE_URL = os.getenv('NEXTCLOUD_BASE_URL').rstrip('/') # Remove trailing slash if any
NEXTCLOUD_BASE_URL = os.getenv('NEXTCLOUD_BASE_URL').rstrip('/')
NEXTCLOUD_USERNAME = os.getenv('NEXTCLOUD_USERNAME')
NEXTCLOUD_PASSWORD = os.getenv('NEXTCLOUD_PASSWORD')
NEXTCLOUD_DIRECTORY = os.getenv('NEXTCLOUD_DIRECTORY').strip('/') # Ensure no leading/trailing slashes
NEXTCLOUD_DIRECTORY = os.getenv('NEXTCLOUD_DIRECTORY').strip('/')
# Load the sync interval from the environment or set a default (in seconds)
SYNC_INTERVAL = int(os.getenv('SYNC_INTERVAL', 60)) # Default is 60 seconds
@@ -24,11 +26,27 @@ local_folders = {
'onboarded_person': 'onboarded_person'
}
def get_nextcloud_files(folder_name):
# Local file tracking for uploads
TRACKING_FILE = 'file_tracking.json'
def load_tracking_data():
"""Load the file tracking data from the tracking file."""
if os.path.exists(TRACKING_FILE):
with open(TRACKING_FILE, 'r') as f:
return json.load(f)
return {}
def save_tracking_data(data):
"""Save the file tracking data to the tracking file."""
with open(TRACKING_FILE, 'w') as f:
json.dump(data, f, indent=4)
def get_nextcloud_files(session, folder_name):
"""
Get a list of files from a specific Nextcloud folder (e.g., 'attachments' or 'onboarded_person').
Get a list of files from a specific Nextcloud folder using a persistent session.
Args:
session (requests.Session): The persistent session object.
folder_name (str): The subfolder name under the group folder on Nextcloud.
Returns:
@@ -39,22 +57,20 @@ def get_nextcloud_files(folder_name):
print(f"Connecting to Nextcloud folder: {nextcloud_url}")
# Make a PROPFIND request to list files in the Nextcloud directory
headers = {
'Depth': '1', # Depth header to get all files in the directory
}
response = requests.request("PROPFIND", nextcloud_url, auth=HTTPBasicAuth(NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD), headers=headers)
headers = {'Depth': '1'}
response = session.request("PROPFIND", nextcloud_url, headers=headers)
if response.status_code != 207:
raise Exception(f"Failed to list Nextcloud files in {folder_name}. Status code: {response.status_code}")
print(f"Successfully retrieved file list from Nextcloud folder: {folder_name}.")
# Extract filenames from the XML response
files = []
for line in response.text.split('\n'):
if '<d:href>' in line:
filename = line.split('<d:href>')[1].split('</d:href>')[0].split('/')[-1]
if filename: # Avoid empty strings
filename = unquote(filename).strip()
if filename:
files.append(filename)
print(f"Nextcloud '{folder_name}' contains {len(files)} files: {files}")
@@ -64,11 +80,12 @@ def get_nextcloud_files(folder_name):
print(f"Error retrieving files from Nextcloud folder {folder_name}: {e}")
return []
def upload_file_to_nextcloud(file_path, filename, folder_name):
def upload_file_to_nextcloud(session, file_path, filename, folder_name):
"""
Upload a file to a specific Nextcloud folder (e.g., 'attachments' or 'onboarded_person').
Upload a file to a specific Nextcloud folder using a persistent session.
Args:
session (requests.Session): The persistent session object.
file_path (str): The full path to the file to upload.
filename (str): The name of the file being uploaded.
folder_name (str): The Nextcloud subfolder where the file should be uploaded.
@@ -78,43 +95,57 @@ def upload_file_to_nextcloud(file_path, filename, folder_name):
print(f"Uploading file: {filename} to Nextcloud folder: {folder_name}...")
with open(file_path, 'rb') as f:
response = requests.put(nextcloud_url, data=f, auth=HTTPBasicAuth(NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD))
response = session.put(nextcloud_url, data=f)
if response.status_code not in [200, 201, 204]: # 204 is a valid status code for success
raise Exception(f"Failed to upload {filename}. Status code: {response.status_code}")
else:
print(f"Successfully uploaded {filename} to Nextcloud folder: {folder_name}.")
return True
except Exception as e:
print(f"Error uploading file {filename} to Nextcloud folder {folder_name}: {e}")
return False
def check_for_new_files(local_folder, nextcloud_files, nextcloud_folder):
def check_for_new_files(session, local_folder, nextcloud_files, nextcloud_folder, tracking_data):
"""
Check for new files in the local folder and upload them to Nextcloud.
Check for new files in the local folder and upload them to Nextcloud using a persistent session.
Args:
session (requests.Session): The persistent session object.
local_folder (str): The local folder to check for new files.
nextcloud_files (list): The list of files already present in the Nextcloud folder.
nextcloud_folder (str): The corresponding Nextcloud folder to upload new files to.
tracking_data (dict): The dictionary containing tracked files and their modification times.
"""
try:
local_files = os.listdir(local_folder)
print(f"Files found in local folder '{local_folder}': {local_files}")
for file in local_files:
if file not in nextcloud_files:
file_path = os.path.join(local_folder, file)
print(f"New file detected: {file} (from {local_folder})")
upload_file_to_nextcloud(file_path, file, nextcloud_folder)
file_mtime = os.path.getmtime(file_path)
# Check if the file is new or modified
if file not in nextcloud_files and (file not in tracking_data or tracking_data[file] != file_mtime):
print(f"New or modified file detected: {file} (from {local_folder})")
if upload_file_to_nextcloud(session, file_path, file, nextcloud_folder):
# Update tracking data after successful upload
tracking_data[file] = file_mtime
else:
print(f"File {file} already exists in Nextcloud folder {nextcloud_folder}. Skipping...")
print(f"File {file} already exists in Nextcloud folder {nextcloud_folder} or has not been modified. Skipping...")
except Exception as e:
print(f"Error checking for new files in {local_folder}: {e}")
def sync_folders():
def sync_folders(session):
"""
Check local folders for new files and upload them to the corresponding Nextcloud folders.
Check local folders for new files and upload them to the corresponding Nextcloud folders using a persistent session.
Args:
session (requests.Session): The persistent session object.
"""
tracking_data = load_tracking_data()
try:
for local_folder, nextcloud_folder in local_folders.items():
print(f"Checking local folder: {local_folder} for new files...")
@@ -126,10 +157,13 @@ def sync_folders():
continue
# Get a list of files in the corresponding Nextcloud folder
nextcloud_files = get_nextcloud_files(nextcloud_folder)
nextcloud_files = get_nextcloud_files(session, nextcloud_folder)
# Check for new files in the local folder and upload them
check_for_new_files(local_folder, nextcloud_files, nextcloud_folder)
check_for_new_files(session, local_folder, nextcloud_files, nextcloud_folder, tracking_data)
# Save updated tracking data
save_tracking_data(tracking_data)
except Exception as e:
print(f"Error during folder synchronization: {e}")
@@ -151,12 +185,17 @@ def start_periodic_sync():
Start the periodic synchronization of local folders to Nextcloud.
The synchronization interval is specified in the .env file or defaults to 60 seconds.
"""
# Use a persistent session for Nextcloud requests
with requests.Session() as session:
# Set up authentication for the session
session.auth = HTTPBasicAuth(NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD)
while True:
print("Starting folder synchronization...")
sync_folders()
sync_folders(session)
print("Synchronization complete.")
countdown_timer(SYNC_INTERVAL) # Wait for the specified sync interval
if __name__ == "__main__":
print("Starting periodic folder synchronization service...")
print("Starting periodic folder synchronization service with a persistent session...")
start_periodic_sync()