DynamoDB support & Serverless config update

requirements.txt

@@ -1,3 +1,5 @@
+boto3==1.23.0
+botocore==1.26.0
 certifi==2021.10.8
 charset-normalizer==2.0.12
 click==8.0.4
@@ -9,12 +11,15 @@ idna==3.3
 importlib-metadata==4.8.3
 itsdangerous==2.0.1
 Jinja2==3.0.3
+jmespath==0.10.0
 MarkupSafe==2.0.1
 pymongo==4.1.1
+python-dateutil==2.8.2
 requests==2.27.1
+s3transfer==0.5.2
 six==1.16.0
 twitter==1.19.3
-typing-extensions==4.1.1
+typing_extensions==4.1.1
 urllib3==1.26.9
 Werkzeug==2.0.3
 youtube-dl==2021.12.17

serverless.yml

@@ -1,8 +1,36 @@
-service: twitfix
+service: vxTwitter
 
 provider:
   name: aws
   runtime: python3.6
+  stage: dev
+  iamRoleStatements:
+    - Effect: Allow
+      Action:
+        - dynamodb:Query
+        - dynamodb:Scan
+        - dynamodb:GetItem
+        - dynamodb:PutItem
+        - dynamodb:UpdateItem
+        - dynamodb:DeleteItem
+      Resource:
+        - { "Fn::GetAtt": ["vxTwitterDynamoTable", "Arn" ] }
+  environment:
+    CACHE_TABLE: ${self:custom.tableName}
+    RUNNING_SERVERLESS: 1
+    VXTWITTER_LINK_CACHE: dynamodb
+    VXTWITTER_DATABASE: none
+    VXTWITTER_DATABASE_TABLE: none
+    VXTWITTER_METHOD: youtube-dl
+    VXTWITTER_COLOR: \#43B581
+    VXTWITTER_APP_NAME: vxTwitter
+    VXTWITTER_REPO: https://github.com/dylanpdx/BetterTwitFix
+    VXTWITTER_URL: https://vxtwitter.com
+    # Twitter API keys
+    VXTWITTER_TWITTER_API_KEY: none
+    VXTWITTER_TWITTER_API_SECRET: none
+    VXTWITTER_TWITTER_ACCESS_TOKEN: none
+    VXTWITTER_TWITTER_ACCESS_SECRET: none
 
 package:
   patterns:
@@ -16,7 +44,7 @@ plugins:
   - serverless-plugin-include-dependencies
 
 functions:
-  webhook:
+  vxTwitterApp:
     handler: wsgi_handler.handler
     url: true
     layers:
@@ -24,8 +52,28 @@ functions:
 
 
 custom:
+  tableName: 'users-table-${self:provider.stage}'
   wsgi:
     app: twitfix.app
   pythonRequirements:
     layer: true
     dockerizePip: true
+
+	
+resources:
+  Resources:
+    vxTwitterDynamoTable:
+      Type: 'AWS::DynamoDB::Table'
+      Properties:
+        AttributeDefinitions:
+          -
+            AttributeName: tweet
+            AttributeType: S
+        KeySchema:
+          -
+            AttributeName: tweet
+            KeyType: HASH
+        ProvisionedThroughput:
+          ReadCapacityUnits: 1
+          WriteCapacityUnits: 1
+        TableName: ${self:custom.tableName}

twitfix.py

@@ -11,6 +11,7 @@ import os
 import urllib.parse
 import urllib.request
 from datetime import date
+import boto3
 
 app = Flask(__name__)
 CORS(app)
@@ -33,26 +34,46 @@ generate_embed_user_agents = [
 
 # Read config from config.json. If it does not exist, create new.
 if not os.path.exists("config.json"):
-    with open("config.json", "w") as outfile:
+    serverless_check = os.environ.get('RUNNING_SERVERLESS')
-        default_config = {
+    if serverless_check == None: # Running on local pc, therefore we can access the filesystem
-            "config":{
+        with open("config.json", "w") as outfile:
-                "link_cache":"json",
+            default_config = {
-                "database":"[url to mongo database goes here]",
+                "config":{
-                "table":"TwiFix",
+                    "link_cache":"dynamodb",
-                "method":"youtube-dl", 
+                    "database":"[url to mongo database goes here]",
-                "color":"#43B581", 
+                    "table":"TwiFix",
-                "appname": "vxTwitter", 
+                    "method":"youtube-dl", 
-                "repo": "https://github.com/dylanpdx/BetterTwitFix", 
+                    "color":"#43B581", 
-                "url": "https://vxtwitter.com"
+                    "appname": "vxTwitter", 
-                },
+                    "repo": "https://github.com/dylanpdx/BetterTwitFix", 
-            "api":{"api_key":"[api_key goes here]",
+                    "url": "https://vxtwitter.com"
-            "api_secret":"[api_secret goes here]",
+                    },
-            "access_token":"[access_token goes here]",
+                "api":{"api_key":"[api_key goes here]",
-            "access_secret":"[access_secret goes here]"
+                "api_secret":"[api_secret goes here]",
+                "access_token":"[access_token goes here]",
+                "access_secret":"[access_secret goes here]"
+                }
+            }
+            json.dump(default_config, outfile, indent=4, sort_keys=True)
+    else: # Running on serverless, therefore we cannot access the filesystem and must use environment variables
+            default_config = {
+                "config":{
+                    "link_cache":os.environ['VXTWITTER_LINK_CACHE'],
+                    "database":os.environ['VXTWITTER_DATABASE'],
+                    "table":os.environ['VXTWITTER_DATABASE_TABLE'],
+                    "method":os.environ['VXTWITTER_METHOD'],
+                    "color":os.environ['VXTWITTER_COLOR'],
+                    "appname": os.environ['VXTWITTER_APP_NAME'],
+                    "repo": os.environ['VXTWITTER_REPO'],
+                    "url": os.environ['VXTWITTER_URL'],
+                    },
+                "api":{
+                    "api_key":os.environ['VXTWITTER_TWITTER_API_KEY'],
+                    "api_secret":os.environ['VXTWITTER_TWITTER_API_SECRET'],
+                    "access_token":os.environ['VXTWITTER_TWITTER_ACCESS_TOKEN'],
+                    "access_secret":os.environ['VXTWITTER_TWITTER_ACCESS_SECRET']
+                }
             }
-        }
-
-        json.dump(default_config, outfile, indent=4, sort_keys=True)
 
     config = default_config
 else:
@@ -66,6 +87,9 @@ if config['config']['method'] in ('api', 'hybrid'):
     twitter_api = twitter.Twitter(auth=auth)
 
 link_cache_system = config['config']['link_cache']
+DYNAMO_CACHE_TBL=None
+if link_cache_system=="dynamodb":
+    DYNAMO_CACHE_TBL=os.environ['CACHE_TABLE']
 
 if link_cache_system == "json":
     link_cache = {}
@@ -81,6 +105,8 @@ elif link_cache_system == "db":
     client = pymongo.MongoClient(config['config']['database'], connect=False)
     table = config['config']['table']
     db = client[table]
+elif link_cache_system == "dynamodb":
+    client = boto3.resource('dynamodb')
 
 @app.route('/') # If the useragent is discord, return the embed, if not, redirect to configured repo directly
 def default():
@@ -381,6 +407,20 @@ def getVnfFromLinkCache(video_link):
         else:
             print(" ➤ [ X ] Link not in json cache")
             return None
+    elif link_cache_system == "dynamodb":
+        table = client.Table(DYNAMO_CACHE_TBL)
+        response = table.get_item(
+            Key={
+                'tweet': video_link
+            }
+        )
+        if 'Item' in response:
+            print("Link located in dynamodb cache")
+            vnf = response['Item']
+            return vnf
+        else:
+            print(" ➤ [ X ] Link not in dynamodb cache")
+            return None
 
 def addVnfToLinkCache(video_link, vnf):
     if link_cache_system == "db":
@@ -396,6 +436,16 @@ def addVnfToLinkCache(video_link, vnf):
         with open("links.json", "w") as outfile: 
             json.dump(link_cache, outfile, indent=4, sort_keys=True)
             return None
+    elif link_cache_system == "dynamodb":
+        table = client.Table(DYNAMO_CACHE_TBL)
+        table.put_item(
+            Item={
+                'tweet': video_link,
+                'vnf': vnf
+            }
+        )
+        print(" ➤ [ + ] Link added to dynamodb cache ")
+        return True
 
 def message(text):
     return render_template(
@@ -407,6 +457,7 @@ def message(text):
         url     = config['config']['url'] )
 
 def embed(video_link, vnf, image):
+    print(vnf)
     print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
     
     desc    = re.sub(r' http.*t\.co\S+', '', vnf['description'])