Project

General

Profile

« Previous | Next » 

Revision 54597

added skip store functionality

View differences:

modules/s3-objectStore-migration/trunk/s3_migration.py
52 52
	client = MongoClient()
53 53
	db = client['objectStore']
54 54
	metadataCollection = db[metadataCollection]
55
	print skip_store
55 56
	for item in metadataCollection.find(no_cursor_timeout=True):
56 57
		obsId = item['obsId']
57 58
		if obsId not in skip_store:
59
			destination_collection.create_index([('id',pymongo.ASCENDING)])
58 60
			exportItemForMongoCollection(obsId, db, bucket, log_file)
59
			destination_collection =db[ 's3_'+obsId[:36]]
60
			print "creating Index on ID"
61
			destination_collection.create_index([('id',pymongo.ASCENDING)])
61
			destination_collection =db[ 's3_'+obsId[:36]]			
62 62
			done_file.write('{}\n'.format(obsId))
63 63

  
64 64

  
......
84 84
	conn = boto.connect_s3(aws_access_key_id = props['access_key'], aws_secret_access_key = props['secret_key'],  host = props['host_base'], calling_format = boto.s3.connection.OrdinaryCallingFormat())
85 85
	bucket = conn.get_bucket(bname, validate=True)
86 86
	log_file = open('s3_migration.log', 'w')
87
	done_file = open('store_done', 'wb')
87
	done_file = open('store_done_1', 'wb')
88 88
	start_import('metadataObjectStore',bucket, log_file, done_file, skip_store)
89 89
	log_file.close()

Also available in: Unified diff