Project

General

Profile

« Previous | Next » 

Revision 54591

fixed check on mongo

View differences:

modules/s3-objectStore-migration/trunk/s3_migration.py
1 1
import boto
2 2
import boto.s3.connection
3 3
import sys
4
import pymongo
4 5
from pymongo import MongoClient
5 6
import os.path
6 7
import time
......
23 24
	for item in source_collection.find(no_cursor_timeout=True):
24 25
		fs_path = item['fsPath']
25 26
		objectId = item['id']
26
		if os.path.isfile(fs_path):
27
			i += 1
28
			if not file_exist(bucket, '%s/%s'%(obsId,objectId)):
29
				key = bucket.new_key('%s/%s'%(obsId,objectId))
30
				try:
31
					key.set_contents_from_filename(fs_path)	
32
				except Exception as e:
33
					time.sleep(10)
34
					key.set_contents_from_filename(fs_path)
35
				
36
				item.pop('_id', None)
37
				item.pop('fsPath')
38
				item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
39
				destination_collection.insert_one(item)
40
			if i % 1000 == 0:
41
				print "Exported %i/%i"%(i, total)
42
		else:
43
			log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
44
	
27
		dest_item = destination_collection.find_one(objectId)
28
		if dest_item is None:
29
			if os.path.isfile(fs_path):
30
				i += 1
31
				if not file_exist(bucket, '%s/%s'%(obsId,objectId)):
32
					key = bucket.new_key('%s/%s'%(obsId,objectId))
33
					try:
34
						key.set_contents_from_filename(fs_path)	
35
					except Exception as e:
36
						time.sleep(10)
37
						key.set_contents_from_filename(fs_path)
38
					
39
					item.pop('_id', None)
40
					item.pop('fsPath')
41
					item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
42
					destination_collection.insert_one(item)
43
				if i % 1000 == 0:
44
					print "Exported %i/%i"%(i, total)
45
			else:
46
				log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
47
		
45 48

  
46 49
def start_import(metadataCollection, bucket, log_file):
47 50
	client = MongoClient()
......
50 53
	for item in metadataCollection.find(no_cursor_timeout=True):
51 54
		obsId = item['obsId']
52 55
		exportItemForMongoCollection(obsId, db, bucket, log_file)
56
		print "creating Index on ID"
57
		destination_collection.create_index([('id',pymongo.ASCENDING)])
53 58

  
54 59

  
55 60

  

Also available in: Unified diff