D-Net

import boto

import boto.s3.connection

import sys

from pymongo import MongoClient

import os.path

def exportItemForMongoCollection(obsId, db, bucket, log_file):	

	destination_collection =db[ 's3_'+obsId[:36]]

	source_collection = db[ obsId[:36]]

	print 'Start to exporting objectStore :%s '%obsId

	i = 0

	last_percentage = 0

	total = source_collection.estimated_document_count()

	for item in source_collection.find(no_cursor_timeout=True):

		fs_path = item['fsPath']

		objectId = item['id']

		if os.path.isfile(fs_path):

			key = bucket.new_key('%s/%s'%(obsId,objectId))

			key.set_contents_from_filename(fs_path)

			i += 1

			item.pop('_id', None)

			item.pop('fsPath')

			item['uri'] = 's3://%s/%s'%(bucket.name, key.name)

			destination_collection.insert_one(item)

			if i % 1000 == 0:

				print "Exported %i/%i"%(i, total)

		else:

			log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))

def start_import(metadataCollection, bucket, log_file):

	client = MongoClient()

	db = client['objectStore']

	metadataCollection = db[metadataCollection]

	for item in metadataCollection.find(no_cursor_timeout=True):

		obsId = item['obsId']

		exportItemForMongoCollection(obsId, db, bucket, log_file)

if __name__=='__main__':

	args = sys.argv

	if not len(args) is not 3 :

		print "Error on applying script usage: python scriptName.py s3cfgPath objectstoreBucket"

	f = open(args[1])

	props = {}	

	for line in f:

		d =line.split('=')

		if len(d) == 2:

			props[d[0].strip()] = d[1].strip()

	bname = args[2]

	conn = boto.connect_s3(aws_access_key_id = props['access_key'], aws_secret_access_key = props['secret_key'],  host = props['host_base'], calling_format = boto.s3.connection.OrdinaryCallingFormat())

	bucket = conn.get_bucket(bname, validate=True)

	log_file = open('s3_migration.log', 'w')

	start_import('metadataObjectStore',bucket, log_file)

	log_file.close()