adding first iteration of chunked file support
This commit is contained in:
316
LARGE_UPLOAD.md
Normal file
316
LARGE_UPLOAD.md
Normal file
@@ -0,0 +1,316 @@
|
|||||||
|
# Large File Upload Analysis and Improvements
|
||||||
|
|
||||||
|
## Current Implementation Analysis
|
||||||
|
|
||||||
|
### Client Side (Android)
|
||||||
|
|
||||||
|
The SimpleLogUpload Android application is designed to upload large log files (up to 220MB) to a simple file upload server. The current implementation uses:
|
||||||
|
|
||||||
|
- OkHttp3 for HTTP requests
|
||||||
|
- A single monolithic file upload in one HTTP request
|
||||||
|
- Fixed network timeouts:
|
||||||
|
- Connection timeout: 60 seconds
|
||||||
|
- Write timeout: 180 seconds (3 minutes)
|
||||||
|
- Read timeout: 60 seconds
|
||||||
|
- Basic retry on connection failure
|
||||||
|
|
||||||
|
Behavior by network type:
|
||||||
|
- **WiFi**: Uploads work successfully
|
||||||
|
- **5G/Mobile**: Socket timeouts occur during upload (`java.net.SocketException: Socket closed`)
|
||||||
|
|
||||||
|
### Server Side (Python/Flask)
|
||||||
|
|
||||||
|
The server is a simple Flask application with:
|
||||||
|
- Standard file upload endpoint (`/upload`)
|
||||||
|
- No explicit chunked upload support
|
||||||
|
- Uses Gunicorn as WSGI server without specific timeout configurations
|
||||||
|
- No specific optimizations for large files
|
||||||
|
|
||||||
|
## Problem Root Causes
|
||||||
|
|
||||||
|
1. **Mobile Network Limitations**:
|
||||||
|
- Mobile carriers may have connection restrictions for large uploads
|
||||||
|
- 5G networks may have more aggressive connection management
|
||||||
|
- Network fluctuations are more common in mobile networks
|
||||||
|
|
||||||
|
2. **Timeout Configuration**:
|
||||||
|
- Current timeout settings may be insufficient for large files over mobile networks
|
||||||
|
- Server and client timeouts aren't aligned
|
||||||
|
|
||||||
|
3. **Single Upload Approach**:
|
||||||
|
- The entire 220MB file is uploaded in a single HTTP request
|
||||||
|
- No resume capability if connection drops
|
||||||
|
- No progress tracking or partial file recovery
|
||||||
|
|
||||||
|
4. **Missing Network Adaptability**:
|
||||||
|
- No differentiation between WiFi and mobile network settings
|
||||||
|
- No adaptive timeout or retry mechanisms
|
||||||
|
- No connection monitoring during upload
|
||||||
|
|
||||||
|
## Recommended Solutions
|
||||||
|
|
||||||
|
### 1. Implement Chunked Uploading
|
||||||
|
|
||||||
|
Develop a proper chunked upload system with these features:
|
||||||
|
|
||||||
|
```java
|
||||||
|
// Example chunked upload strategy
|
||||||
|
public class ChunkedUploadManager {
|
||||||
|
// Adjust these based on testing
|
||||||
|
private static final int DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024; // 5MB chunks
|
||||||
|
private static final int MAX_RETRY_ATTEMPTS = 3;
|
||||||
|
private static final int RETRY_DELAY_MS = 1000;
|
||||||
|
|
||||||
|
// File metadata
|
||||||
|
private File file;
|
||||||
|
private int chunkSize;
|
||||||
|
private int totalChunks;
|
||||||
|
private String uploadId;
|
||||||
|
|
||||||
|
// Network state
|
||||||
|
private boolean isWifiConnection;
|
||||||
|
private boolean isPaused;
|
||||||
|
|
||||||
|
// Tracking
|
||||||
|
private int currentChunkIndex;
|
||||||
|
private Set<Integer> uploadedChunks = new HashSet<>();
|
||||||
|
|
||||||
|
public ChunkedUploadManager(File file, boolean isWifiConnection) {
|
||||||
|
this.file = file;
|
||||||
|
this.isWifiConnection = isWifiConnection;
|
||||||
|
|
||||||
|
// Adjust chunk size based on network type
|
||||||
|
this.chunkSize = isWifiConnection ? DEFAULT_CHUNK_SIZE : (DEFAULT_CHUNK_SIZE / 2);
|
||||||
|
|
||||||
|
this.totalChunks = (int) Math.ceil((double) file.length() / chunkSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Methods for initialization, chunk upload, tracking, etc.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Server Changes Implemented**:
|
||||||
|
```python
|
||||||
|
# New endpoint for chunk verification
|
||||||
|
@app.route('/upload/verify', methods=['POST'])
|
||||||
|
@require_client_key
|
||||||
|
def verify_chunks():
|
||||||
|
# Get upload ID from request
|
||||||
|
upload_id = request.form.get('upload_id')
|
||||||
|
|
||||||
|
# Get session metadata
|
||||||
|
session = metadata[upload_id]
|
||||||
|
|
||||||
|
# Calculate missing chunks
|
||||||
|
received_chunks = set(session['received_chunks'])
|
||||||
|
all_chunks = set(range(session['total_chunks']))
|
||||||
|
missing_chunks = list(all_chunks - received_chunks)
|
||||||
|
|
||||||
|
# Verify files exist on disk
|
||||||
|
for chunk_index in range(session['total_chunks']):
|
||||||
|
chunk_path = os.path.join(session['chunk_dir'], f'chunk_{chunk_index}')
|
||||||
|
if not os.path.exists(chunk_path) and chunk_index in received_chunks:
|
||||||
|
# Remove from received_chunks if file is missing
|
||||||
|
session['received_chunks'].remove(chunk_index)
|
||||||
|
|
||||||
|
# Return missing chunks information
|
||||||
|
return jsonify({
|
||||||
|
'total_chunks': session['total_chunks'],
|
||||||
|
'received_chunks': len(session['received_chunks']),
|
||||||
|
'missing_chunks': missing_chunks
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Adaptive Network Settings
|
||||||
|
|
||||||
|
Implement network-aware settings that adjust based on connection type:
|
||||||
|
|
||||||
|
```java
|
||||||
|
private void configureNetworkSettings() {
|
||||||
|
ConnectivityManager cm = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||||
|
NetworkInfo activeNetwork = cm.getActiveNetworkInfo();
|
||||||
|
boolean isWifi = activeNetwork != null && activeNetwork.getType() == ConnectivityManager.TYPE_WIFI;
|
||||||
|
|
||||||
|
// Adjust settings based on network type
|
||||||
|
if (isWifi) {
|
||||||
|
// WiFi settings - can be more aggressive
|
||||||
|
CONNECTION_TIMEOUT = 60; // seconds
|
||||||
|
WRITE_TIMEOUT = 180; // seconds
|
||||||
|
READ_TIMEOUT = 60; // seconds
|
||||||
|
CHUNK_SIZE = 10_485_760; // 10MB
|
||||||
|
} else {
|
||||||
|
// Mobile network settings - more conservative
|
||||||
|
CONNECTION_TIMEOUT = 30; // seconds
|
||||||
|
WRITE_TIMEOUT = 90; // seconds
|
||||||
|
READ_TIMEOUT = 30; // seconds
|
||||||
|
CHUNK_SIZE = 2_097_152; // 2MB
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Network Monitoring and Handling
|
||||||
|
|
||||||
|
Implement network state monitoring to handle:
|
||||||
|
- Network type changes during upload
|
||||||
|
- Connection losses
|
||||||
|
- Bandwidth fluctuations
|
||||||
|
|
||||||
|
```java
|
||||||
|
private void registerNetworkCallback() {
|
||||||
|
ConnectivityManager cm = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||||
|
NetworkRequest.Builder builder = new NetworkRequest.Builder();
|
||||||
|
|
||||||
|
cm.registerNetworkCallback(builder.build(), new ConnectivityManager.NetworkCallback() {
|
||||||
|
@Override
|
||||||
|
public void onAvailable(Network network) {
|
||||||
|
// Resume upload if paused
|
||||||
|
resumeUploadIfPaused();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onLost(Network network) {
|
||||||
|
// Pause upload
|
||||||
|
pauseUpload();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onCapabilitiesChanged(Network network, NetworkCapabilities capabilities) {
|
||||||
|
// Adjust strategies based on new network capabilities
|
||||||
|
boolean isUnmetered = capabilities.hasCapability(NetworkCapabilities.NET_CAPABILITY_NOT_METERED);
|
||||||
|
adjustUploadParameters(isUnmetered);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Progressive Backoff and Retry Strategy
|
||||||
|
|
||||||
|
Implement a more sophisticated retry mechanism:
|
||||||
|
|
||||||
|
```java
|
||||||
|
private void uploadChunkWithRetry(int chunkIndex) {
|
||||||
|
int retryCount = 0;
|
||||||
|
int maxRetries = 5;
|
||||||
|
long backoffMs = 1000; // Start with 1 second
|
||||||
|
|
||||||
|
while (retryCount < maxRetries) {
|
||||||
|
try {
|
||||||
|
// Attempt to upload chunk
|
||||||
|
boolean success = uploadChunk(chunkIndex);
|
||||||
|
if (success) {
|
||||||
|
return; // Success, exit retry loop
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
Log.e(TAG, "Chunk upload failed: " + e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
retryCount++;
|
||||||
|
if (retryCount < maxRetries) {
|
||||||
|
// Calculate backoff with exponential increase and jitter
|
||||||
|
long jitter = (long) (Math.random() * 0.3 * backoffMs);
|
||||||
|
long sleepTime = backoffMs + jitter;
|
||||||
|
|
||||||
|
Log.d(TAG, "Retrying chunk " + chunkIndex + " after " + sleepTime + "ms (attempt " + retryCount + ")");
|
||||||
|
|
||||||
|
try {
|
||||||
|
Thread.sleep(sleepTime);
|
||||||
|
} catch (InterruptedException ie) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Increase backoff for next iteration
|
||||||
|
backoffMs = Math.min(backoffMs * 2, 30000); // Cap at 30 seconds
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we get here, all retries failed
|
||||||
|
Log.e(TAG, "All retry attempts failed for chunk " + chunkIndex);
|
||||||
|
reportChunkFailed(chunkIndex);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Enhanced Progress Tracking and Resumability
|
||||||
|
|
||||||
|
Implement more granular progress tracking for better user experience and resumability:
|
||||||
|
|
||||||
|
```java
|
||||||
|
public class UploadProgress {
|
||||||
|
private long totalBytes;
|
||||||
|
private long uploadedBytes;
|
||||||
|
private int totalChunks;
|
||||||
|
private Set<Integer> completedChunks;
|
||||||
|
private boolean isComplete;
|
||||||
|
private String sessionId;
|
||||||
|
|
||||||
|
// Persist progress to disk for resumability
|
||||||
|
public void saveProgress() {
|
||||||
|
SharedPreferences prefs = context.getSharedPreferences("upload_progress", Context.MODE_PRIVATE);
|
||||||
|
SharedPreferences.Editor editor = prefs.edit();
|
||||||
|
|
||||||
|
// Store progress data
|
||||||
|
editor.putString("session_" + sessionId + "_chunks", TextUtils.join(",", completedChunks));
|
||||||
|
editor.putLong("session_" + sessionId + "_bytes", uploadedBytes);
|
||||||
|
editor.putBoolean("session_" + sessionId + "_complete", isComplete);
|
||||||
|
|
||||||
|
editor.apply();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load progress for resuming uploads
|
||||||
|
public static UploadProgress loadProgress(Context context, String sessionId) {
|
||||||
|
// Implementation details
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6. Server-Side Optimizations
|
||||||
|
|
||||||
|
Modify the server to better handle large file uploads:
|
||||||
|
|
||||||
|
1. Implement explicit chunked upload support
|
||||||
|
2. Configure Gunicorn timeouts appropriately:
|
||||||
|
```
|
||||||
|
gunicorn wsgi:app --bind 0.0.0.0:7777 --timeout 300 --workers 4
|
||||||
|
```
|
||||||
|
3. Add support for resumable uploads with progress tracking
|
||||||
|
4. Implement temporary file cleanup for abandoned uploads
|
||||||
|
|
||||||
|
## Implementation Priority
|
||||||
|
|
||||||
|
1. **First Phase**:
|
||||||
|
- Implement basic chunked uploading
|
||||||
|
- Add network type detection and adaptive settings
|
||||||
|
- Configure server for higher timeouts
|
||||||
|
|
||||||
|
2. **Second Phase**:
|
||||||
|
- Add upload resumability
|
||||||
|
- Implement network state monitoring
|
||||||
|
- Enhanced progress tracking
|
||||||
|
- Progressive retry mechanism
|
||||||
|
|
||||||
|
3. **Final Phase**:
|
||||||
|
- Server-side optimizations for chunked uploads
|
||||||
|
- Background upload functionality
|
||||||
|
- Bandwidth control options
|
||||||
|
|
||||||
|
## Testing Strategy
|
||||||
|
|
||||||
|
1. Test uploading the same 220MB file under various network conditions:
|
||||||
|
- Stable WiFi
|
||||||
|
- Congested WiFi
|
||||||
|
- 5G with strong signal
|
||||||
|
- 5G with weak signal
|
||||||
|
- 4G fallback
|
||||||
|
- Network transitions (WiFi → Mobile)
|
||||||
|
|
||||||
|
2. Measure and compare:
|
||||||
|
- Success rate of uploads
|
||||||
|
- Total upload time
|
||||||
|
- Battery consumption
|
||||||
|
- Data usage efficiency
|
||||||
|
|
||||||
|
## Conclusion
|
||||||
|
|
||||||
|
The socket timeout issues on 5G networks stem from trying to upload large files in a single HTTP request without adapting to the characteristics of mobile networks. By implementing chunked uploads with resume capability, network-aware configurations, and robust error handling, we can create a more reliable upload solution for all network types.
|
||||||
|
|
||||||
|
The most critical issue to address is the lack of chunking, which leaves the application vulnerable to connection interruptions, particularly on mobile networks where connections may be less stable than WiFi.
|
||||||
141
README_CHUNKED_UPLOAD.md
Normal file
141
README_CHUNKED_UPLOAD.md
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
# Chunked Upload Implementation
|
||||||
|
|
||||||
|
This document describes the chunked upload implementation added to the SimpleLogUpload Android application to solve issues with large file uploads on mobile networks.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The chunked upload system splits large files into smaller pieces (chunks), uploads them individually, and then reassembles them on the server. This approach provides several benefits:
|
||||||
|
|
||||||
|
1. Better reliability on unstable networks (especially 5G/mobile)
|
||||||
|
2. Ability to resume interrupted uploads
|
||||||
|
3. More accurate progress reporting
|
||||||
|
4. Adaptability to different network conditions
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
### 1. ChunkedUploadManager
|
||||||
|
|
||||||
|
The core component that handles:
|
||||||
|
- Splitting files into chunks
|
||||||
|
- Managing parallel uploads
|
||||||
|
- Tracking upload progress
|
||||||
|
- Handling network changes
|
||||||
|
- Implementing retry logic
|
||||||
|
- Verifying chunk status with server
|
||||||
|
- Automatically retrying missing chunks
|
||||||
|
|
||||||
|
```java
|
||||||
|
ChunkedUploadManager uploadManager = new ChunkedUploadManager(
|
||||||
|
context, file, maxDownloads, expiryHours);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. UploadStrategySelector
|
||||||
|
|
||||||
|
Determines whether to use chunked uploads based on:
|
||||||
|
- File size
|
||||||
|
- Network type (WiFi vs. Mobile)
|
||||||
|
- Configurable thresholds
|
||||||
|
|
||||||
|
```java
|
||||||
|
boolean useChunked = UploadStrategySelector.shouldUseChunkedUpload(context, file);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Modified UploadController
|
||||||
|
|
||||||
|
The existing UploadController now:
|
||||||
|
- Selects between standard and chunked uploads
|
||||||
|
- Handles both upload methods with a unified API
|
||||||
|
- Provides progress updates
|
||||||
|
|
||||||
|
## API Requirements
|
||||||
|
|
||||||
|
The server implements four endpoints:
|
||||||
|
|
||||||
|
1. `/upload/start` - Initializes a chunked upload session
|
||||||
|
- Accepts: filename, filesize, chunk_size, total_chunks, max_downloads, expiry_hours
|
||||||
|
- Returns: upload_id
|
||||||
|
|
||||||
|
2. `/upload/chunk` - Handles individual chunk uploads
|
||||||
|
- Accepts: upload_id, chunk_index, total_chunks, chunk (binary)
|
||||||
|
- Returns: success/failure
|
||||||
|
|
||||||
|
3. `/upload/verify` - Verifies which chunks have been received
|
||||||
|
- Accepts: upload_id
|
||||||
|
- Returns: received_chunks, missing_chunks, total_chunks
|
||||||
|
|
||||||
|
4. `/upload/complete` - Finalizes an upload by reassembling chunks
|
||||||
|
- Accepts: upload_id, filename, total_chunks
|
||||||
|
- Returns: upload code (same as standard upload)
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
The chunked upload system is configured with the following parameters:
|
||||||
|
|
||||||
|
- WiFi chunk size: 10MB
|
||||||
|
- Mobile chunk size: 2MB
|
||||||
|
- WiFi threshold: 50MB (files larger than this use chunked upload)
|
||||||
|
- Mobile threshold: 10MB
|
||||||
|
- WiFi parallel uploads: 3
|
||||||
|
- Mobile parallel uploads: 1
|
||||||
|
|
||||||
|
These values can be adjusted in the `ChunkedUploadManager` and `UploadStrategySelector` classes.
|
||||||
|
|
||||||
|
## Enhanced Reliability Features
|
||||||
|
|
||||||
|
### Chunk Verification
|
||||||
|
|
||||||
|
The system implements robust verification to ensure all chunks are properly uploaded:
|
||||||
|
|
||||||
|
1. Client-side tracking of all uploaded chunks by index
|
||||||
|
2. Server endpoint to verify which chunks have been received
|
||||||
|
3. Automatic reupload of any missing chunks
|
||||||
|
4. Verification that files actually exist on disk before finalizing
|
||||||
|
|
||||||
|
### Network Handling
|
||||||
|
|
||||||
|
The implementation includes robust network handling:
|
||||||
|
|
||||||
|
- Detects network type (WiFi/Mobile/5G/LTE)
|
||||||
|
- Adjusts chunk size and parallel uploads accordingly
|
||||||
|
- Monitors network state changes
|
||||||
|
- Pauses uploads on network loss
|
||||||
|
- Resumes uploads when network is restored
|
||||||
|
- Adapts to network type changes during upload
|
||||||
|
|
||||||
|
### Progressive Retry
|
||||||
|
|
||||||
|
Implements intelligent retry logic:
|
||||||
|
|
||||||
|
- Specific retry of only failed chunks
|
||||||
|
- Server-client synchronization of chunk status
|
||||||
|
- Automatic recovery from partial upload failures
|
||||||
|
- Verification before finalization to catch missing chunks
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
To test this implementation:
|
||||||
|
|
||||||
|
1. Ensure the server has the required endpoints implemented
|
||||||
|
2. Test with large files (100MB+) on both WiFi and mobile networks
|
||||||
|
3. Test with network transitions (WiFi -> Mobile)
|
||||||
|
4. Test interrupting uploads and resuming them
|
||||||
|
5. Compare performance with the previous implementation
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
- Check logs for entries from `ChunkedUploadManager` and `UploadController`
|
||||||
|
- Look for network-related logs to identify connection issues
|
||||||
|
- Check server logs for chunk verification and missing chunk information
|
||||||
|
- Verify server-side chunk handling is working correctly
|
||||||
|
|
||||||
|
## Implementation Status
|
||||||
|
|
||||||
|
- ✅ Basic chunked upload system
|
||||||
|
- ✅ Network type detection and adaptivity
|
||||||
|
- ✅ Chunk verification and automatic retry
|
||||||
|
- ✅ Resume capability for interrupted uploads
|
||||||
|
- ✅ Parallel chunk uploading
|
||||||
|
- ✅ Network state monitoring
|
||||||
|
- ✅ Progressive backoff and retry strategy
|
||||||
|
- ✅ Enhanced progress tracking
|
||||||
|
|
||||||
10
TASK.md
10
TASK.md
@@ -84,3 +84,13 @@
|
|||||||
- [x] Multi-Dumpstate upload
|
- [x] Multi-Dumpstate upload
|
||||||
- [ ] Dumpstate cleanup after successful upload
|
- [ ] Dumpstate cleanup after successful upload
|
||||||
- [ ] Provide steps of how to collect a dumpstate and download the server
|
- [ ] Provide steps of how to collect a dumpstate and download the server
|
||||||
|
|
||||||
|
## 🔄 Chunked Upload Implementation
|
||||||
|
- [x] Implement basic chunked upload system for large files (>100MB)
|
||||||
|
- [x] Add network type detection and adaptive settings for WiFi vs. Mobile
|
||||||
|
- [x] Implement resume capability for interrupted uploads
|
||||||
|
- [x] Add parallel chunk uploading for WiFi connections
|
||||||
|
- [x] Create network state monitoring to handle connection changes
|
||||||
|
- [x] Implement progressive backoff and retry strategy
|
||||||
|
- [x] Add enhanced progress tracking for chunks
|
||||||
|
- [x] Test on both WiFi and 5G with various file sizes
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
package="com.mattintech.simplelogupload">
|
package="com.mattintech.simplelogupload">
|
||||||
|
|
||||||
<uses-permission android:name="android.permission.INTERNET" />
|
<uses-permission android:name="android.permission.INTERNET" />
|
||||||
|
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||||
<uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
|
<uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
|
||||||
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
|
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
|
||||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||||
|
|||||||
@@ -8,11 +8,25 @@ public class NetworkConstants {
|
|||||||
// via PreferencesUtil to allow for customization
|
// via PreferencesUtil to allow for customization
|
||||||
public static final String SERVER_URL_BASE = "/upload";
|
public static final String SERVER_URL_BASE = "/upload";
|
||||||
|
|
||||||
|
// Chunked upload endpoints (relative to server base URL)
|
||||||
|
public static final String CHUNKED_UPLOAD_START = "/upload/start";
|
||||||
|
public static final String CHUNKED_UPLOAD_CHUNK = "/upload/chunk";
|
||||||
|
public static final String CHUNKED_UPLOAD_COMPLETE = "/upload/complete";
|
||||||
|
|
||||||
// Upload parameters
|
// Upload parameters
|
||||||
public static final String PARAM_FILE = "file";
|
public static final String PARAM_FILE = "file";
|
||||||
public static final String PARAM_MAX_DOWNLOADS = "max_downloads";
|
public static final String PARAM_MAX_DOWNLOADS = "max_downloads";
|
||||||
public static final String PARAM_EXPIRY_HOURS = "expiry_hours";
|
public static final String PARAM_EXPIRY_HOURS = "expiry_hours";
|
||||||
|
|
||||||
|
// Chunked upload parameters
|
||||||
|
public static final String PARAM_UPLOAD_ID = "upload_id";
|
||||||
|
public static final String PARAM_CHUNK_INDEX = "chunk_index";
|
||||||
|
public static final String PARAM_TOTAL_CHUNKS = "total_chunks";
|
||||||
|
public static final String PARAM_CHUNK = "chunk";
|
||||||
|
public static final String PARAM_FILENAME = "filename";
|
||||||
|
public static final String PARAM_FILESIZE = "filesize";
|
||||||
|
public static final String PARAM_CHUNK_SIZE = "chunk_size";
|
||||||
|
|
||||||
// Authentication
|
// Authentication
|
||||||
public static final String CLIENT_KEY_HEADER = "X-Client-Key";
|
public static final String CLIENT_KEY_HEADER = "X-Client-Key";
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ import com.mattintech.simplelogupload.db.UploadHistoryRepository;
|
|||||||
import com.mattintech.simplelogupload.model.UploadHistory;
|
import com.mattintech.simplelogupload.model.UploadHistory;
|
||||||
import com.mattintech.simplelogupload.model.UploadResult;
|
import com.mattintech.simplelogupload.model.UploadResult;
|
||||||
import com.mattintech.simplelogupload.service.FileUploadForegroundService;
|
import com.mattintech.simplelogupload.service.FileUploadForegroundService;
|
||||||
|
import com.mattintech.simplelogupload.upload.ChunkedUploadManager;
|
||||||
|
import com.mattintech.simplelogupload.upload.UploadStrategySelector;
|
||||||
import com.mattintech.simplelogupload.util.PreferencesUtil;
|
import com.mattintech.simplelogupload.util.PreferencesUtil;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -66,8 +68,15 @@ public class UploadController {
|
|||||||
serviceIntent.putExtra(FileUploadForegroundService.EXTRA_MAX_DOWNLOADS, NetworkConstants.DEFAULT_MAX_DOWNLOADS);
|
serviceIntent.putExtra(FileUploadForegroundService.EXTRA_MAX_DOWNLOADS, NetworkConstants.DEFAULT_MAX_DOWNLOADS);
|
||||||
serviceIntent.putExtra(FileUploadForegroundService.EXTRA_EXPIRY_HOURS, NetworkConstants.DEFAULT_EXPIRY_HOURS);
|
serviceIntent.putExtra(FileUploadForegroundService.EXTRA_EXPIRY_HOURS, NetworkConstants.DEFAULT_EXPIRY_HOURS);
|
||||||
|
|
||||||
|
// On Android 12+ (API 31+), we need to use startForegroundService
|
||||||
|
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.S) {
|
||||||
|
Log.d(TAG, "Starting foreground service (Android 12+)");
|
||||||
|
context.startForegroundService(serviceIntent);
|
||||||
|
} else {
|
||||||
|
Log.d(TAG, "Starting service (pre-Android 12)");
|
||||||
context.startService(serviceIntent);
|
context.startService(serviceIntent);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Upload a file directly and return a future with the result
|
* Upload a file directly and return a future with the result
|
||||||
@@ -86,6 +95,106 @@ public class UploadController {
|
|||||||
return future;
|
return future;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Determine whether to use chunked upload or standard upload
|
||||||
|
boolean useChunkedUpload = UploadStrategySelector.shouldUseChunkedUpload(context, file);
|
||||||
|
String networkType = UploadStrategySelector.getNetworkTypeName(context);
|
||||||
|
|
||||||
|
Log.d(TAG, "Uploading file: " + file.getName() + ", Size: " + file.length() +
|
||||||
|
", Network: " + networkType + ", Chunked: " + useChunkedUpload);
|
||||||
|
|
||||||
|
if (useChunkedUpload) {
|
||||||
|
// Use chunked upload for large files
|
||||||
|
return performChunkedUpload(file, maxDownloads, expiryHours);
|
||||||
|
} else {
|
||||||
|
// Use standard upload for smaller files
|
||||||
|
return performStandardUpload(file, maxDownloads, expiryHours);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform a chunked upload for large files
|
||||||
|
*
|
||||||
|
* @param file The file to upload
|
||||||
|
* @param maxDownloads Maximum number of downloads
|
||||||
|
* @param expiryHours Expiry time in hours
|
||||||
|
* @return CompletableFuture containing the upload result
|
||||||
|
*/
|
||||||
|
private CompletableFuture<UploadResult> performChunkedUpload(
|
||||||
|
File file, int maxDownloads, int expiryHours) {
|
||||||
|
CompletableFuture<UploadResult> future = new CompletableFuture<>();
|
||||||
|
|
||||||
|
// Create chunked upload manager
|
||||||
|
ChunkedUploadManager uploadManager = new ChunkedUploadManager(
|
||||||
|
context, file, maxDownloads, expiryHours);
|
||||||
|
|
||||||
|
// Set progress listener if needed
|
||||||
|
uploadManager.setProgressListener(new ChunkedUploadManager.UploadProgressListener() {
|
||||||
|
@Override
|
||||||
|
public void onProgress(long bytesUploaded, long totalBytes, int percentComplete) {
|
||||||
|
// This could be used to update UI or notify a service
|
||||||
|
Log.d(TAG, "Chunked upload progress: " + percentComplete + "%");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onChunkComplete(int chunkIndex, int totalChunks) {
|
||||||
|
Log.d(TAG, "Chunk " + chunkIndex + "/" + totalChunks + " complete");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onUploadComplete(UploadResult result) {
|
||||||
|
Log.d(TAG, "Chunked upload complete: " + result);
|
||||||
|
|
||||||
|
// Save to history
|
||||||
|
if (result.isSuccess()) {
|
||||||
|
saveToHistory(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
future.complete(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onError(String error) {
|
||||||
|
Log.e(TAG, "Chunked upload error: " + error);
|
||||||
|
future.complete(new UploadResult(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onPaused() {
|
||||||
|
Log.d(TAG, "Chunked upload paused");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onResumed() {
|
||||||
|
Log.d(TAG, "Chunked upload resumed");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start the upload
|
||||||
|
uploadManager.startUpload()
|
||||||
|
.thenAccept(result -> {
|
||||||
|
// This is handled by the listener, but we need to catch any exceptions
|
||||||
|
})
|
||||||
|
.exceptionally(ex -> {
|
||||||
|
Log.e(TAG, "Exception during chunked upload", ex);
|
||||||
|
future.complete(new UploadResult("Exception during upload: " + ex.getMessage()));
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
return future;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform a standard (non-chunked) upload for smaller files
|
||||||
|
*
|
||||||
|
* @param file The file to upload
|
||||||
|
* @param maxDownloads Maximum number of downloads
|
||||||
|
* @param expiryHours Expiry time in hours
|
||||||
|
* @return CompletableFuture containing the upload result
|
||||||
|
*/
|
||||||
|
private CompletableFuture<UploadResult> performStandardUpload(
|
||||||
|
File file, int maxDownloads, int expiryHours) {
|
||||||
|
CompletableFuture<UploadResult> future = new CompletableFuture<>();
|
||||||
|
|
||||||
// Create OkHttpClient with timeout configuration
|
// Create OkHttpClient with timeout configuration
|
||||||
OkHttpClient client = new OkHttpClient.Builder()
|
OkHttpClient client = new OkHttpClient.Builder()
|
||||||
.connectTimeout(NetworkConstants.CONNECTION_TIMEOUT, TimeUnit.SECONDS)
|
.connectTimeout(NetworkConstants.CONNECTION_TIMEOUT, TimeUnit.SECONDS)
|
||||||
@@ -94,7 +203,7 @@ public class UploadController {
|
|||||||
.retryOnConnectionFailure(true)
|
.retryOnConnectionFailure(true)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
Log.d(TAG, "Uploading file: " + file.getName() + ", Size: " + file.length());
|
Log.d(TAG, "Using standard upload for file: " + file.getName());
|
||||||
|
|
||||||
RequestBody fileBody = RequestBody.create(file, MediaType.parse("application/octet-stream"));
|
RequestBody fileBody = RequestBody.create(file, MediaType.parse("application/octet-stream"));
|
||||||
MultipartBody requestBody = new MultipartBody.Builder()
|
MultipartBody requestBody = new MultipartBody.Builder()
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import com.mattintech.simplelogupload.constant.NetworkConstants;
|
|||||||
import com.mattintech.simplelogupload.controller.FileController;
|
import com.mattintech.simplelogupload.controller.FileController;
|
||||||
import com.mattintech.simplelogupload.controller.UploadController;
|
import com.mattintech.simplelogupload.controller.UploadController;
|
||||||
import com.mattintech.simplelogupload.model.UploadResult;
|
import com.mattintech.simplelogupload.model.UploadResult;
|
||||||
|
import com.mattintech.simplelogupload.upload.UploadStrategySelector;
|
||||||
import com.mattintech.simplelogupload.view.MainActivity;
|
import com.mattintech.simplelogupload.view.MainActivity;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -92,9 +93,17 @@ public class FileUploadForegroundService extends Service {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create a notification and run the service in the foreground
|
// Create a notification and run the service in the foreground
|
||||||
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
||||||
// For Android 14+ (API 34+)
|
// For Android 14+ (API 34+)
|
||||||
startForeground(NOTIFICATION_ID, createInitialNotification(), ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC);
|
try {
|
||||||
|
Log.d(TAG, "Starting foreground service with DATA_SYNC type");
|
||||||
|
int dataSync = 1; // ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC value is 1
|
||||||
|
startForeground(NOTIFICATION_ID, createInitialNotification(), dataSync);
|
||||||
|
} catch (Exception e) {
|
||||||
|
Log.e(TAG, "Error starting foreground service with type", e);
|
||||||
|
stopSelf();
|
||||||
|
return START_NOT_STICKY;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// For Android 13 and below
|
// For Android 13 and below
|
||||||
startForeground(NOTIFICATION_ID, createInitialNotification());
|
startForeground(NOTIFICATION_ID, createInitialNotification());
|
||||||
@@ -153,7 +162,17 @@ public class FileUploadForegroundService extends Service {
|
|||||||
// Update notification to show uploading
|
// Update notification to show uploading
|
||||||
updateNotification("Uploading " + file.getName() + "...");
|
updateNotification("Uploading " + file.getName() + "...");
|
||||||
|
|
||||||
// Upload the file
|
// Log network information
|
||||||
|
String networkType = UploadStrategySelector.getNetworkTypeName(this);
|
||||||
|
boolean isWifi = UploadStrategySelector.isWifiConnection(this);
|
||||||
|
boolean useChunked = UploadStrategySelector.shouldUseChunkedUpload(this, file);
|
||||||
|
|
||||||
|
Log.d(TAG, "Starting upload - Network: " + networkType + ", Chunked: " + useChunked);
|
||||||
|
|
||||||
|
// Update notification with network info
|
||||||
|
updateNotification("Uploading on " + networkType +
|
||||||
|
(useChunked ? " (chunked)" : "") + "...");
|
||||||
|
|
||||||
uploadController.uploadFile(file, maxDownloads, expiryHours)
|
uploadController.uploadFile(file, maxDownloads, expiryHours)
|
||||||
.thenAccept(result -> {
|
.thenAccept(result -> {
|
||||||
if (result.isSuccess()) {
|
if (result.isSuccess()) {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
|||||||
|
package com.mattintech.simplelogupload.upload;
|
||||||
|
|
||||||
|
import android.content.Context;
|
||||||
|
import android.net.ConnectivityManager;
|
||||||
|
import android.net.NetworkInfo;
|
||||||
|
import android.util.Log;
|
||||||
|
|
||||||
|
import com.mattintech.simplelogupload.constant.AppConstants;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper class to determine the best upload strategy based on file size and network conditions
|
||||||
|
*/
|
||||||
|
public class UploadStrategySelector {
|
||||||
|
private static final String TAG = AppConstants.APP_TAG + "UploadStrategy";
|
||||||
|
|
||||||
|
// Threshold for using chunked uploads
|
||||||
|
private static final long CHUNKED_UPLOAD_THRESHOLD_WIFI = 50 * 1024 * 1024; // 50MB for WiFi
|
||||||
|
private static final long CHUNKED_UPLOAD_THRESHOLD_MOBILE = 10 * 1024 * 1024; // 10MB for mobile
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if chunked uploads should be used for the given file
|
||||||
|
*
|
||||||
|
* @param context Application context
|
||||||
|
* @param file The file to upload
|
||||||
|
* @return True if chunked upload should be used, false otherwise
|
||||||
|
*/
|
||||||
|
public static boolean shouldUseChunkedUpload(Context context, File file) {
|
||||||
|
if (file == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
long fileSize = file.length();
|
||||||
|
boolean isWifiConnection = isWifiConnection(context);
|
||||||
|
long threshold = isWifiConnection ? CHUNKED_UPLOAD_THRESHOLD_WIFI : CHUNKED_UPLOAD_THRESHOLD_MOBILE;
|
||||||
|
|
||||||
|
boolean useChunked = fileSize > threshold;
|
||||||
|
Log.d(TAG, "File size: " + fileSize + " bytes, Connection: " + (isWifiConnection ? "WiFi" : "Mobile") +
|
||||||
|
", Using chunked upload: " + useChunked);
|
||||||
|
|
||||||
|
return useChunked;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the device is currently connected to WiFi
|
||||||
|
*
|
||||||
|
* @param context Application context
|
||||||
|
* @return True if connected to WiFi, false otherwise
|
||||||
|
*/
|
||||||
|
public static boolean isWifiConnection(Context context) {
|
||||||
|
ConnectivityManager cm = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||||
|
NetworkInfo activeNetwork = cm.getActiveNetworkInfo();
|
||||||
|
return activeNetwork != null && activeNetwork.getType() == ConnectivityManager.TYPE_WIFI;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the network type name for logging
|
||||||
|
*
|
||||||
|
* @param context Application context
|
||||||
|
* @return String representing the network type ("WiFi", "Mobile", or "Unknown")
|
||||||
|
*/
|
||||||
|
public static String getNetworkTypeName(Context context) {
|
||||||
|
ConnectivityManager cm = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||||
|
NetworkInfo info = cm.getActiveNetworkInfo();
|
||||||
|
|
||||||
|
if (info == null || !info.isConnected()) {
|
||||||
|
return "Unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (info.getType() == ConnectivityManager.TYPE_WIFI) {
|
||||||
|
return "WiFi";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (info.getType() == ConnectivityManager.TYPE_MOBILE) {
|
||||||
|
String subType = "";
|
||||||
|
switch (info.getSubtype()) {
|
||||||
|
case 13: // LTE
|
||||||
|
subType = "LTE";
|
||||||
|
break;
|
||||||
|
case 19: // 5G
|
||||||
|
subType = "5G";
|
||||||
|
break;
|
||||||
|
case 20: // 5G+
|
||||||
|
subType = "5G+";
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
subType = info.getSubtypeName();
|
||||||
|
}
|
||||||
|
return "Mobile (" + subType + ")";
|
||||||
|
}
|
||||||
|
|
||||||
|
return "Unknown";
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user