Revision 54591
Added by Sandro La Bruzzo about 5 years ago
modules/s3-objectStore-migration/trunk/s3_migration.py | ||
---|---|---|
1 | 1 |
import boto |
2 | 2 |
import boto.s3.connection |
3 | 3 |
import sys |
4 |
import pymongo |
|
4 | 5 |
from pymongo import MongoClient |
5 | 6 |
import os.path |
6 | 7 |
import time |
... | ... | |
23 | 24 |
for item in source_collection.find(no_cursor_timeout=True): |
24 | 25 |
fs_path = item['fsPath'] |
25 | 26 |
objectId = item['id'] |
26 |
if os.path.isfile(fs_path): |
|
27 |
i += 1 |
|
28 |
if not file_exist(bucket, '%s/%s'%(obsId,objectId)): |
|
29 |
key = bucket.new_key('%s/%s'%(obsId,objectId)) |
|
30 |
try: |
|
31 |
key.set_contents_from_filename(fs_path) |
|
32 |
except Exception as e: |
|
33 |
time.sleep(10) |
|
34 |
key.set_contents_from_filename(fs_path) |
|
35 |
|
|
36 |
item.pop('_id', None) |
|
37 |
item.pop('fsPath') |
|
38 |
item['uri'] = 's3://%s/%s'%(bucket.name, key.name) |
|
39 |
destination_collection.insert_one(item) |
|
40 |
if i % 1000 == 0: |
|
41 |
print "Exported %i/%i"%(i, total) |
|
42 |
else: |
|
43 |
log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path)) |
|
44 |
|
|
27 |
dest_item = destination_collection.find_one(objectId) |
|
28 |
if dest_item is None: |
|
29 |
if os.path.isfile(fs_path): |
|
30 |
i += 1 |
|
31 |
if not file_exist(bucket, '%s/%s'%(obsId,objectId)): |
|
32 |
key = bucket.new_key('%s/%s'%(obsId,objectId)) |
|
33 |
try: |
|
34 |
key.set_contents_from_filename(fs_path) |
|
35 |
except Exception as e: |
|
36 |
time.sleep(10) |
|
37 |
key.set_contents_from_filename(fs_path) |
|
38 |
|
|
39 |
item.pop('_id', None) |
|
40 |
item.pop('fsPath') |
|
41 |
item['uri'] = 's3://%s/%s'%(bucket.name, key.name) |
|
42 |
destination_collection.insert_one(item) |
|
43 |
if i % 1000 == 0: |
|
44 |
print "Exported %i/%i"%(i, total) |
|
45 |
else: |
|
46 |
log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path)) |
|
47 |
|
|
45 | 48 |
|
46 | 49 |
def start_import(metadataCollection, bucket, log_file): |
47 | 50 |
client = MongoClient() |
... | ... | |
50 | 53 |
for item in metadataCollection.find(no_cursor_timeout=True): |
51 | 54 |
obsId = item['obsId'] |
52 | 55 |
exportItemForMongoCollection(obsId, db, bucket, log_file) |
56 |
print "creating Index on ID" |
|
57 |
destination_collection.create_index([('id',pymongo.ASCENDING)]) |
|
53 | 58 |
|
54 | 59 |
|
55 | 60 |
|
Also available in: Unified diff
fixed check on mongo