{"id":232,"date":"2022-04-12T09:00:27","date_gmt":"2022-04-12T00:00:27","guid":{"rendered":"https:\/\/is-ai.jp\/?p=232"},"modified":"2022-04-13T16:39:23","modified_gmt":"2022-04-13T07:39:23","slug":"aws-dynamodb%e3%81%ae%e3%83%87%e3%83%bc%e3%82%bf%e3%82%92csv%e3%83%95%e3%82%a1%e3%82%a4%e3%83%ab%e3%81%a7%e5%87%ba%e5%8a%9b%e3%81%99%e3%82%8b","status":"publish","type":"post","link":"https:\/\/is-ai.jp\/?p=232","title":{"rendered":"AWS dynamoDB\u306e\u30c7\u30fc\u30bf\u3092csv\u30d5\u30a1\u30a4\u30eb\u3067\u51fa\u529b\u3059\u308b"},"content":{"rendered":"\n<p><\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u306f\u3058\u3081\u306b<\/h4>\n\n\n\n<p><strong>dynamoDB\u306e\u7d50\u679c\u3092\u5b9a\u671f\u7684\u306b\u78ba\u8a8d\u3057\u305f\u3044\u3002<\/strong>\u3068\u3044\u3046\u8981\u671b\u304c\u3042\u308a\u3001\u305d\u306e\u5b9f\u88c5\u3092\u884c\u3044\u307e\u3057\u305f\u3002\u57fa\u672c\u7684\u306a\u65b9\u91dd\u306fAWS Lambda\u3067\u5b9a\u671f\u5b9f\u884c\u3092\u884c\u3044\u3001dynamoDB\u5185\u306e\u30c7\u30fc\u30bf\u3092csv\u5f62\u5f0f\u306b\u4fdd\u5b58\u3057\u3066\u3001google drive\u306b\u30a2\u30c3\u30d7\u30ed\u30fc\u30c9\u3059\u308b\u3068\u3044\u3046\u6d41\u308c\u3067\u3059\u3002<\/p>\n\n\n\n<p><\/p>\n\n\n\n<p>dynamoDB\u306e\u30c7\u30fc\u30bf\u3092csv\u7b49\u306b\u51fa\u529b\u3059\u308b\u3053\u3068\u306fAWS\u7ba1\u7406\u753b\u9762\u304b\u3089\u306e\u64cd\u4f5c\u3060\u3051\u3067\u306f\u3067\u304d\u307e\u305b\u3093\u3002\uff08\u8868\u793a\u3057\u3066\u3044\u308b\u5206\u3057\u304b\u51fa\u529b\u3067\u304d\u306a\u3044\u3068\u3044\u3046\u5236\u9650\u304c\u5b58\u5728\u3059\u308b\uff09\u307e\u305f\u3001\u5b9a\u671f\u5b9f\u884c\u3082\u4f7f\u3044\u305f\u304b\u3063\u305f\u306e\u3067\u4eca\u56de\u306fpython\u3068AWS Lambda\u3092\u4f7f\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<p><\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u5b9f\u88c5<\/h4>\n\n\n\n<p>dynamoDB\u306e\u30c6\u30fc\u30d6\u30eb\u304b\u3089\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\u3057\u3066\u3044\u304d\u307e\u3059\u3002GUI\u64cd\u4f5c\u3067\u306f\u5236\u9650\u304c\u3042\u308b\u3068\u66f8\u304d\u307e\u3057\u305f\u304c\u3001\u30b3\u30fc\u30c9\u304b\u3089\u306e\u64cd\u4f5c\u3067\u3082\u5b9f\u306f\u5236\u9650\u304c\u3042\u308a\u30015000\u4ef6\u307e\u3067\u3057\u304b\u53d6\u5f97\u3067\u304d\u307e\u305b\u3093\u3002\u305d\u3053\u3067\u3001\u30eb\u30fc\u30d7\u51e6\u7406\u3092\u52a0\u3048\u3066\u5168\u3066\u306e\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\u3057\u3066\u3044\u304d\u307e\u3059\u3002<\/p>\n\n\n\n<div class=\"hcb_wrap\"><pre class=\"prism line-numbers lang-python\" data-lang=\"Python\"><code>def get_all_data(dynamodb, table_name):\n    response = dynamodb.scan(TableName=table_name)\n    data = response[&#39;Items&#39;]\n\n    # \u30ec\u30b9\u30dd\u30f3\u30b9\u306b LastEvaluatedKey \u304c\u542b\u307e\u308c\u306a\u304f\u306a\u308b\u307e\u3067\u30eb\u30fc\u30d7\u51e6\u7406\u3092\u5b9f\u884c\u3059\u308b\n    while &#39;LastEvaluatedKey&#39; in response:\n        response = dynamodb.scan(TableName=table_name, ExclusiveStartKey=response[&#39;LastEvaluatedKey&#39;])\n        if &#39;LastEvaluatedKey&#39; in response:\n            print(&quot;LastEvaluatedKey: {}&quot;.format(response[&#39;LastEvaluatedKey&#39;]))\n        data.extend(response[&#39;Items&#39;])\n\n    return data<\/code><\/pre><\/div>\n\n\n\n<p><\/p>\n\n\n\n<p>\u6b21\u306bgoogle drive\u3078\u306e\u30a2\u30c3\u30d7\u30ed\u30fc\u30c9\u306f\u4ee5\u4e0b\u306e\u30b3\u30fc\u30c9\u3067\u884c\u3044\u307e\u3059\u3002&lt;&lt;<strong>your_parents<\/strong>&gt;&gt;\u306b\u306f\u4fdd\u5b58\u3057\u305f\u3044drive\u306eid\u540d\u3092\u8a2d\u5b9a\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<p>\u307e\u305f\u3001\u540c\u4e00\u968e\u5c64\u306b\u30b5\u30fc\u30d3\u30b9\u30a2\u30ab\u30a6\u30f3\u30c8\u30ad\u30fc\u3092\u914d\u7f6e\u3057\u3066\u304f\u3060\u3055\u3044\u3002<\/p>\n\n\n\n<p><\/p>\n\n\n\n<p>google drive\u306b\u30a2\u30c3\u30d7\u30ed\u30fc\u30c9\u3059\u308b\u305f\u3081\u306b\u306f API\u3092\u4f7f\u3048\u308b\u3088\u3046\u306b\u8a2d\u5b9a\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002google drive API\u30ad\u30fc\u306e\u53d6\u5f97\u65b9\u6cd5\u306f<a rel=\"noreferrer noopener\" href=\"https:\/\/cloud.google.com\/iam\/docs\/creating-managing-service-account-keys?hl=ja\" data-type=\"URL\" data-id=\"https:\/\/cloud.google.com\/iam\/docs\/creating-managing-service-account-keys?hl=ja\" target=\"_blank\">\u3053\u3061\u3089<\/a>\u304c\u53c2\u8003\u306b\u306a\u308a\u307e\u3059\u3002\u4eca\u56de\u306fgoogle drive\u3092\u30a2\u30c3\u30d7\u30ed\u30fc\u30c9\u5148\u306b\u9078\u3093\u3067\u3044\u307e\u3059\u304c\u3001\u4efb\u610f\u306e\u74b0\u5883\u306b\u5909\u66f4\u3057\u3066\u304f\u3060\u3055\u3044\u3002<\/p>\n\n\n\n<div class=\"hcb_wrap\"><pre class=\"prism line-numbers lang-python\" data-lang=\"Python\"><code>def uploadFileToGoogleDrive(fileName, localFilePath):\n    ext = os.path.splitext(localFilePath.lower())[1][1:]\n    if ext == &quot;jpg&quot;:\n        ext = &quot;jpeg&quot;\n    mimeType = &quot;image\/&quot; + ext\n\n    service = getGoogleService()\n    file_metadata = {&quot;name&quot;: fileName, &quot;mimeType&quot;: mimeType, &quot;parents&quot;: [&lt;&lt;your_parents&gt;&gt;] } \n    media = MediaFileUpload(localFilePath, mimetype=mimeType, resumable=True) \n    file = service.files().create(body=file_metadata, media_body=media, fields=&#39;id&#39;).execute()\n\n\ndef getGoogleService():\n    scope = [&#39;https:\/\/www.googleapis.com\/auth\/drive.file&#39;] \n    keyFile = &#39;service-account-key.json&#39;\n    credentials = ServiceAccountCredentials.from_json_keyfile_name(keyFile, scopes=scope)\n\n    return build(&quot;drive&quot;, &quot;v3&quot;, credentials=credentials, cache_discovery=False)\n<\/code><\/pre><\/div>\n\n\n\n<p><\/p>\n\n\n\n<p>&lt;&lt;table_name&gt;&gt;\u3068&lt;&lt;column_names&gt;&gt;\u306b\u306f\u4fdd\u5b58\u3057\u305f\u3044DynamoDB\u306e\u30c6\u30fc\u30d6\u30eb\u60c5\u5831\u3092\u5165\u529b\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<div class=\"hcb_wrap\"><pre class=\"prism line-numbers lang-python\" data-lang=\"Python\"><code>import boto3\nfrom boto3.session import Session\nfrom googleapiclient.discovery import build \nfrom googleapiclient.http import MediaFileUpload \nfrom oauth2client.service_account import ServiceAccountCredentials \nimport csv\nimport json\nimport os\nimport ast\nimport datetime\n\nimport logging\nlogger = logging.getLogger()\nlogger.setLevel(logging.INFO)\n\ndef lambda_handler(event, context):\n    session = Session()\n    dynamo = session.client(&quot;dynamodb&quot;)\n    \n    table_names = [&lt;&lt;table_name&gt;&gt;]\n    labels_list = [[&lt;&lt;column_names&gt;&gt;]]\n    \n    dt_now = datetime.datetime.now()\n    \n    for table_name, labels in zip (table_names, labels_list):\n        get_data = get_all_data(dynamo, table_name)\n\n        with open(&#39;\/tmp\/upload.csv&#39;, &#39;w&#39;, encoding=&#39;utf_8_sig&#39;) as f:\n            writer = csv.DictWriter(f, fieldnames=labels)\n            writer.writeheader()\n            for elem in get_data:\n                keys = elem.keys()\n                values = elem.values()\n                values_list = []\n                \n                for v in values:\n                    dic = ast.literal_eval(str(v))\n                    \n                    if type(list(dic.values())[0]) is list:\n                        lis = [list(dic.values())[0][0][&#39;N&#39;], list(dic.values())[0][1][&#39;N&#39;], list(dic.values())[0][2][&#39;N&#39;]]\n                        values_list.append(lis)\n                        \n                    else:\n                        print(list(dic.values())[0])\n                        values_list.append(list(dic.values())[0])\n                \n                d = dict(zip(keys,values_list))\n                writer.writerow(d)\n        \n        uploadFileToGoogleDrive(f&#39;{table_name}_{dt_now.year}_{dt_now.month}_{dt_now.day}.csv&#39;, &#39;\/tmp\/upload.csv&#39;)\n        \n    return<\/code><\/pre><\/div>\n\n\n\n<p><\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u7d50\u679c<\/h4>\n\n\n\n<p>\u7121\u4e8b\u3001google drive\u306b\u30a2\u30c3\u30d7\u30ed\u30fc\u30c9\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3057\u305f\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image size-large is-resized\"><img decoding=\"async\" loading=\"lazy\" src=\"https:\/\/is-ai.jp\/wp-content\/uploads\/2022\/03\/3658183c591bcfe057d463406397ffab-1024x604.png\" alt=\"\" class=\"wp-image-235\" width=\"714\" height=\"420\" srcset=\"https:\/\/is-ai.jp\/wp-content\/uploads\/2022\/03\/3658183c591bcfe057d463406397ffab-1024x604.png 1024w, https:\/\/is-ai.jp\/wp-content\/uploads\/2022\/03\/3658183c591bcfe057d463406397ffab-300x177.png 300w, https:\/\/is-ai.jp\/wp-content\/uploads\/2022\/03\/3658183c591bcfe057d463406397ffab-768x453.png 768w, https:\/\/is-ai.jp\/wp-content\/uploads\/2022\/03\/3658183c591bcfe057d463406397ffab-1536x906.png 1536w, https:\/\/is-ai.jp\/wp-content\/uploads\/2022\/03\/3658183c591bcfe057d463406397ffab-2048x1208.png 2048w\" sizes=\"(max-width: 714px) 100vw, 714px\" \/><\/figure>\n\n\n\n<h4 class=\"wp-block-heading\">\u304a\u308f\u308a\u306b<\/h4>\n\n\n\n<p>\u4ee5\u4e0a\u3067\u4efb\u610f\u306e\u30c6\u30fc\u30d6\u30eb\u30c7\u30fc\u30bf\u3092csv\u5f62\u5f0f\u3067google drive\u306b\u51fa\u529b\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3057\u305f\u3002\u30c7\u30fc\u30bf\u306e\u5171\u6709\u306f\u5fc5\u8981\u306a\u5de5\u7a0b\u3060\u3068\u601d\u3046\u306e\u3067\u3001\u4eca\u5f8c\u3082\u51fa\u756a\u304c\u3042\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u306f\u3058\u3081\u306b dynamoDB\u306e\u7d50\u679c\u3092\u5b9a\u671f\u7684\u306b\u78ba\u8a8d\u3057\u305f\u3044\u3002\u3068\u3044\u3046\u8981\u671b\u304c\u3042\u308a\u3001\u305d\u306e\u5b9f\u88c5\u3092\u884c\u3044\u307e\u3057\u305f\u3002\u57fa\u672c\u7684\u306a\u65b9\u91dd\u306fAWS Lambda\u3067\u5b9a\u671f\u5b9f\u884c\u3092\u884c\u3044\u3001dynamoDB\u5185\u306e\u30c7\u30fc\u30bf\u3092csv\u5f62\u5f0f\u306b\u4fdd\u5b58\u3057\u3066\u3001google drive\u2026<\/p>\n","protected":false},"author":1,"featured_media":233,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[10],"tags":[20,21,7],"_links":{"self":[{"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/posts\/232"}],"collection":[{"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/is-ai.jp\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=232"}],"version-history":[{"count":5,"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/posts\/232\/revisions"}],"predecessor-version":[{"id":239,"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/posts\/232\/revisions\/239"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/is-ai.jp\/index.php?rest_route=\/wp\/v2\/media\/233"}],"wp:attachment":[{"href":"https:\/\/is-ai.jp\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=232"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/is-ai.jp\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=232"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/is-ai.jp\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=232"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}