Project

General

Profile

« Previous | Next » 

Revision 54586

resume migration of s3_objectStore

View differences:

modules/s3-objectStore-migration/trunk/s3_migration.py
3 3
import sys
4 4
from pymongo import MongoClient
5 5
import os.path
6
import time
6 7

  
7 8

  
9

  
10
def file_exist(bucket, file_path):
11
	return len(list(bucket.objects.filter(Prefix=file_path))) > 0
12

  
8 13
def exportItemForMongoCollection(obsId, db, bucket, log_file):	
9 14
	destination_collection =db[ 's3_'+obsId[:36]]
10 15
	source_collection = db[ obsId[:36]]
......
16 21
		fs_path = item['fsPath']
17 22
		objectId = item['id']
18 23
		if os.path.isfile(fs_path):
19
			key = bucket.new_key('%s/%s'%(obsId,objectId))
20
			key.set_contents_from_filename(fs_path)
21
			i += 1
22
			item.pop('_id', None)
23
			item.pop('fsPath')
24
			item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
25
			destination_collection.insert_one(item)
24
			if not file_exist(bucket, '%s/%s'%(obsId,objectId)):
25
				key = bucket.new_key('%s/%s'%(obsId,objectId))
26
				try:
27
					key.set_contents_from_filename(fs_path)	
28
				except Exception as e:
29
					time.sleep(10)
30
					key.set_contents_from_filename(fs_path)
31
				i += 1
32
				item.pop('_id', None)
33
				item.pop('fsPath')
34
				item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
35
				destination_collection.insert_one(item)
26 36
			if i % 1000 == 0:
27 37
				print "Exported %i/%i"%(i, total)
28 38
		else:
29 39
			log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
30 40
	
31 41

  
32

  
33

  
34

  
35

  
36

  
37

  
38

  
39 42
def start_import(metadataCollection, bucket, log_file):
40 43
	client = MongoClient()
41 44
	db = client['objectStore']

Also available in: Unified diff