Revision 54586
Added by Sandro La Bruzzo over 5 years ago
| s3_migration.py | ||
|---|---|---|
| 3 | 3 |
import sys |
| 4 | 4 |
from pymongo import MongoClient |
| 5 | 5 |
import os.path |
| 6 |
import time |
|
| 6 | 7 |
|
| 7 | 8 |
|
| 9 |
|
|
| 10 |
def file_exist(bucket, file_path): |
|
| 11 |
return len(list(bucket.objects.filter(Prefix=file_path))) > 0 |
|
| 12 |
|
|
| 8 | 13 |
def exportItemForMongoCollection(obsId, db, bucket, log_file): |
| 9 | 14 |
destination_collection =db[ 's3_'+obsId[:36]] |
| 10 | 15 |
source_collection = db[ obsId[:36]] |
| ... | ... | |
| 16 | 21 |
fs_path = item['fsPath'] |
| 17 | 22 |
objectId = item['id'] |
| 18 | 23 |
if os.path.isfile(fs_path): |
| 19 |
key = bucket.new_key('%s/%s'%(obsId,objectId))
|
|
| 20 |
key.set_contents_from_filename(fs_path) |
|
| 21 |
i += 1 |
|
| 22 |
item.pop('_id', None)
|
|
| 23 |
item.pop('fsPath')
|
|
| 24 |
item['uri'] = 's3://%s/%s'%(bucket.name, key.name) |
|
| 25 |
destination_collection.insert_one(item) |
|
| 24 |
if not file_exist(bucket, '%s/%s'%(obsId,objectId)): |
|
| 25 |
key = bucket.new_key('%s/%s'%(obsId,objectId))
|
|
| 26 |
try: |
|
| 27 |
key.set_contents_from_filename(fs_path) |
|
| 28 |
except Exception as e: |
|
| 29 |
time.sleep(10) |
|
| 30 |
key.set_contents_from_filename(fs_path) |
|
| 31 |
i += 1 |
|
| 32 |
item.pop('_id', None)
|
|
| 33 |
item.pop('fsPath')
|
|
| 34 |
item['uri'] = 's3://%s/%s'%(bucket.name, key.name) |
|
| 35 |
destination_collection.insert_one(item) |
|
| 26 | 36 |
if i % 1000 == 0: |
| 27 | 37 |
print "Exported %i/%i"%(i, total) |
| 28 | 38 |
else: |
| 29 | 39 |
log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
|
| 30 | 40 |
|
| 31 | 41 |
|
| 32 |
|
|
| 33 |
|
|
| 34 |
|
|
| 35 |
|
|
| 36 |
|
|
| 37 |
|
|
| 38 |
|
|
| 39 | 42 |
def start_import(metadataCollection, bucket, log_file): |
| 40 | 43 |
client = MongoClient() |
| 41 | 44 |
db = client['objectStore'] |
Also available in: Unified diff
resume migration of s3_objectStore