Fixed DynamoDB integration

commit fba1256efd
Author: Dylan <dylanpdx@gmail.com>
Date:   Sun May 15 01:42:22 2022 +0100

    Fixed infinite redirect(?)

commit 85c682ab11
Author: Dylan <dylanpdx@gmail.com>
Date:   Sun May 15 01:15:15 2022 +0100

    Updated gitignore

.gitignore

@@ -2,3 +2,9 @@ config.json
 __pycache__
 venv/
 links.json
+node_modules
+#SERVERLESS STUFF
+admin.env
+.env
+_meta
+.serverless

package.json

@@ -0,0 +1,28 @@
+{
+  "name": "bettertwitfix",
+  "version": "1.0.0",
+  "description": "(A fork of TwitFix)\r Basic flask server that serves fixed twitter video embeds to desktop discord by using either the Twitter API or Youtube-DL to grab tweet video information. This also automatically embeds the first link in the text of non video tweets (API Only)",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dylanpdx/BetterTwitFix.git"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/dylanpdx/BetterTwitFix/issues"
+  },
+  "homepage": "https://github.com/dylanpdx/BetterTwitFix#readme",
+  "dependencies": {
+    "serverless-wsgi": "^3.0.0"
+  },
+  "devDependencies": {
+    "serverless-plugin-common-excludes": "^4.0.0",
+    "serverless-plugin-include-dependencies": "^5.0.0",
+    "serverless-python-requirements": "^5.4.0"
+  }
+}

serverless.yml

@@ -0,0 +1,79 @@
+service: vxTwitter
+
+provider:
+  name: aws
+  runtime: python3.6
+  stage: dev
+  iamRoleStatements:
+    - Effect: Allow
+      Action:
+        - dynamodb:Query
+        - dynamodb:Scan
+        - dynamodb:GetItem
+        - dynamodb:PutItem
+        - dynamodb:UpdateItem
+        - dynamodb:DeleteItem
+      Resource:
+        - { "Fn::GetAtt": ["vxTwitterDynamoTable", "Arn" ] }
+  environment:
+    CACHE_TABLE: ${self:custom.tableName}
+    RUNNING_SERVERLESS: 1
+    VXTWITTER_LINK_CACHE: dynamodb
+    VXTWITTER_DATABASE: none
+    VXTWITTER_DATABASE_TABLE: none
+    VXTWITTER_METHOD: youtube-dl
+    VXTWITTER_COLOR: \#43B581
+    VXTWITTER_APP_NAME: vxTwitter
+    VXTWITTER_REPO: https://github.com/dylanpdx/BetterTwitFix
+    VXTWITTER_URL: https://vxtwitter.com
+    # Twitter API keys
+    VXTWITTER_TWITTER_API_KEY: none
+    VXTWITTER_TWITTER_API_SECRET: none
+    VXTWITTER_TWITTER_ACCESS_TOKEN: none
+    VXTWITTER_TWITTER_ACCESS_SECRET: none
+
+package:
+  patterns:
+    - '!node_modules/**'
+    - '!venv/**'
+
+plugins:
+  - serverless-wsgi
+  - serverless-python-requirements
+  - serverless-plugin-common-excludes
+  - serverless-plugin-include-dependencies
+
+functions:
+  vxTwitterApp:
+    handler: wsgi_handler.handler
+    url: true
+    layers:
+      - Ref: PythonRequirementsLambdaLayer
+
+
+custom:
+  tableName: 'users-table-${self:provider.stage}'
+  wsgi:
+    app: twitfix.app
+  pythonRequirements:
+    layer: true
+    dockerizePip: true
+
+	
+resources:
+  Resources:
+    vxTwitterDynamoTable:
+      Type: 'AWS::DynamoDB::Table'
+      Properties:
+        AttributeDefinitions:
+          -
+            AttributeName: tweet
+            AttributeType: S
+        KeySchema:
+          -
+            AttributeName: tweet
+            KeyType: HASH
+        ProvisionedThroughput:
+          ReadCapacityUnits: 1
+          WriteCapacityUnits: 1
+        TableName: ${self:custom.tableName}

templates/image.html

@@ -14,22 +14,17 @@
     {{ pfp }}     - URL of the posters PFP
  -->
 {% block head %}
-    <meta content="text/html; charset=UTF-8"        http-equiv="Content-Type" />
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
-    <meta content="{{ color }}"                     name="theme-color" />
+<meta content="{{ color }}" name="theme-color" />
-    <meta property="og:site_name"                   content="{{ appname }}"/>
+<meta property="og:site_name" content="{{ appname }}" />
 
-    <meta name="twitter:card"                       content="summary_large_image" />
+<meta name="twitter:card" content="summary_large_image" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta name="twitter:creator"                    content="@{{ user }}" />
+<meta name="twitter:creator" content="@{{ user }}" />
 
-    <meta property="og:description"                 content="{{ desc }}"  />
+<meta property="og:description" content="{{ desc }}" />
 
-    <!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
+<!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

templates/index.html

@@ -1,37 +1,30 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}">
 
-{% block head %}
+<meta name="twitter:card" content="player" />
-    <meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta name="twitter:title" content="{{ user }}" />
-    <meta content="{{ color }}" name="theme-color" />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta property="og:site_name" content="{{ appname }}">
+<!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
-
-    <meta name="twitter:card"                       content="player" />
-    <meta name="twitter:title"                      content="{{ user }}" />
-    <meta name="twitter:image"                      content="{{ pic }}" />
-    <!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
     <meta name="twitter:player"                     content="{{ vidurl }}" />
     <meta name="twitter:description"                content="{{ desc }}" />!-->
-    <meta name="twitter:player:width"               content="720" />
+<meta name="twitter:player:width" content="720" />
-    <meta name="twitter:player:height"              content="480" />
+<meta name="twitter:player:height" content="480" />
-    <meta name="twitter:player:stream"              content="{{ vidurl }}" />
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
-    <meta name="twitter:player:stream:content_type" content="video/mp4" /> 
+<meta name="twitter:player:stream:content_type" content="video/mp4" />
 
 
-    <meta property="og:url"                content="{{ vidlink }}" />
+<meta property="og:url" content="{{ vidlink }}" />
-    <meta property="og:video"              content="{{ vidurl }}" />
+<meta property="og:video" content="{{ vidurl }}" />
-    <meta property="og:video:secure_url"   content="{{ vidurl }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
-    <meta property="og:video:type"         content="video/mp4" />
+<meta property="og:video:type" content="video/mp4" />
-    <meta property="og:video:width"        content="720" />
+<meta property="og:video:width" content="720" />
-    <meta property="og:video:height"       content="480" />
+<meta property="og:video:height" content="480" />
-    <meta property="og:title"              content="{{ user }}" />
+<meta property="og:title" content="{{ user }}" />
-    <!--<meta property="og:description"        content="{{ desc }}" />!-->
+<!--<meta property="og:description"        content="{{ desc }}" />!-->
-    <meta property="og:image"              content="{{ pic }}" />
+<meta property="og:image" content="{{ pic }}" />
 
-        <!--!-->
+<!--!-->
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

templates/text.html

@@ -18,22 +18,17 @@
     {{ time }}    - Time Created
  -->
 {% block head %}
-    <meta content="text/html; charset=UTF-8"        http-equiv="Content-Type" />
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
-    <meta content="{{ color }}"                     name="theme-color" />
+<meta content="{{ color }}" name="theme-color" />
-    <meta property="og:site_name"                   content="{{ appname }}"/>
+<meta property="og:site_name" content="{{ appname }}" />
-    <meta property="og:image"                       content="{{ pfp }}"   />
+<meta property="og:image" content="{{ pfp }}" />
 
-    <meta name="twitter:card"                       content="tweet"       />
+<meta name="twitter:card" content="tweet" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta name="twitter:creator"                    content="@{{ user }}" />
+<meta name="twitter:creator" content="@{{ user }}" />
 
-    <meta property="og:description"                 content="{{ desc }}"  />
+<meta property="og:description" content="{{ desc }}" />
 
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

templates/video.html

@@ -1,31 +1,24 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}">
 
-{% block head %}
+<meta name="twitter:card" content="player" />
-    <meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
-    <meta content="{{ color }}"  name="theme-color" />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta property="og:site_name" content="{{ appname }}">
+<meta name="twitter:player:width" content="720" />
+<meta name="twitter:player:height" content="480" />
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
+<meta name="twitter:player:stream:content_type" content="video/mp4" />
 
-    <meta name="twitter:card"                       content="player" />
+<meta property="og:url" content="{{ vidlink }}" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
+<meta property="og:video" content="{{ vidurl }}" />
-    <meta name="twitter:image"                      content="{{ pic }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
-    <meta name="twitter:player:width"               content="720" />
+<meta property="og:video:type" content="video/mp4" />
-    <meta name="twitter:player:height"              content="480" />
+<meta property="og:video:width" content="720" />
-    <meta name="twitter:player:stream"              content="{{ vidurl }}" />
+<meta property="og:video:height" content="480" />
-    <meta name="twitter:player:stream:content_type" content="video/mp4" /> 
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
+<meta property="og:image" content="{{ pic }}" />
 
-    <meta property="og:url"                content="{{ vidlink }}" />
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
-    <meta property="og:video"              content="{{ vidurl }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-    <meta property="og:video:secure_url"   content="{{ vidurl }}" />
-    <meta property="og:video:type"         content="video/mp4" />
-    <meta property="og:video:width"        content="720" />
-    <meta property="og:video:height"       content="480" />
-    <meta name="twitter:title"             content="{{ user }} (@{{ screenName }})"  />
-    <meta property="og:image"              content="{{ pic }}" />
-
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

twitfix.py

@@ -11,6 +11,7 @@ import os
 import urllib.parse
 import urllib.request
 from datetime import date
+import boto3
 
 app = Flask(__name__)
 CORS(app)
@@ -33,10 +34,12 @@ generate_embed_user_agents = [
 
 # Read config from config.json. If it does not exist, create new.
 if not os.path.exists("config.json"):
+    serverless_check = os.environ.get('RUNNING_SERVERLESS')
+    if serverless_check == None: # Running on local pc, therefore we can access the filesystem
         with open("config.json", "w") as outfile:
             default_config = {
                 "config":{
-                "link_cache":"json",
+                    "link_cache":"dynamodb",
                     "database":"[url to mongo database goes here]",
                     "table":"TwiFix",
                     "method":"youtube-dl", 
@@ -51,8 +54,26 @@ if not os.path.exists("config.json"):
                 "access_secret":"[access_secret goes here]"
                 }
             }
-
             json.dump(default_config, outfile, indent=4, sort_keys=True)
+    else: # Running on serverless, therefore we cannot access the filesystem and must use environment variables
+            default_config = {
+                "config":{
+                    "link_cache":os.environ['VXTWITTER_LINK_CACHE'],
+                    "database":os.environ['VXTWITTER_DATABASE'],
+                    "table":os.environ['VXTWITTER_DATABASE_TABLE'],
+                    "method":os.environ['VXTWITTER_METHOD'],
+                    "color":os.environ['VXTWITTER_COLOR'],
+                    "appname": os.environ['VXTWITTER_APP_NAME'],
+                    "repo": os.environ['VXTWITTER_REPO'],
+                    "url": os.environ['VXTWITTER_URL'],
+                    },
+                "api":{
+                    "api_key":os.environ['VXTWITTER_TWITTER_API_KEY'],
+                    "api_secret":os.environ['VXTWITTER_TWITTER_API_SECRET'],
+                    "access_token":os.environ['VXTWITTER_TWITTER_ACCESS_TOKEN'],
+                    "access_secret":os.environ['VXTWITTER_TWITTER_ACCESS_SECRET']
+                }
+            }
 
     config = default_config
 else:
@@ -66,6 +87,9 @@ if config['config']['method'] in ('api', 'hybrid'):
     twitter_api = twitter.Twitter(auth=auth)
 
 link_cache_system = config['config']['link_cache']
+DYNAMO_CACHE_TBL=None
+if link_cache_system=="dynamodb":
+    DYNAMO_CACHE_TBL=os.environ['CACHE_TABLE']
 
 if link_cache_system == "json":
     link_cache = {}
@@ -81,6 +105,8 @@ elif link_cache_system == "db":
     client = pymongo.MongoClient(config['config']['database'], connect=False)
     table = config['config']['table']
     db = client[table]
+elif link_cache_system == "dynamodb":
+    client = boto3.resource('dynamodb')
 
 @app.route('/') # If the useragent is discord, return the embed, if not, redirect to configured repo directly
 def default():
@@ -266,7 +292,7 @@ def link_to_vnf_from_api(video_link):
     twid = int(re.sub(r'\?.*$','',video_link.rsplit("/", 1)[-1])) # gets the tweet ID as a int from the passed url
     tweet = twitter_api.statuses.show(_id=twid, tweet_mode="extended")
     # For when I need to poke around and see what a tweet looks like
-    #print(tweet)
+    print(tweet)
     imgs = ["","","","", ""]
     print(" ➤ [ + ] Tweet Type: " + tweetType(tweet))
     # Check to see if tweet has a video, if not, make the url passed to the VNF the first t.co link in the tweet
@@ -381,6 +407,20 @@ def getVnfFromLinkCache(video_link):
         else:
             print(" ➤ [ X ] Link not in json cache")
             return None
+    elif link_cache_system == "dynamodb":
+        table = client.Table(DYNAMO_CACHE_TBL)
+        response = table.get_item(
+            Key={
+                'tweet': video_link
+            }
+        )
+        if 'Item' in response:
+            print("Link located in dynamodb cache")
+            vnf = response['Item']['vnf']
+            return vnf
+        else:
+            print(" ➤ [ X ] Link not in dynamodb cache")
+            return None
 
 def addVnfToLinkCache(video_link, vnf):
     if link_cache_system == "db":
@@ -396,6 +436,16 @@ def addVnfToLinkCache(video_link, vnf):
         with open("links.json", "w") as outfile: 
             json.dump(link_cache, outfile, indent=4, sort_keys=True)
             return None
+    elif link_cache_system == "dynamodb":
+        table = client.Table(DYNAMO_CACHE_TBL)
+        table.put_item(
+            Item={
+                'tweet': video_link,
+                'vnf': vnf
+            }
+        )
+        print(" ➤ [ + ] Link added to dynamodb cache ")
+        return True
 
 def message(text):
     return render_template(
@@ -407,6 +457,7 @@ def message(text):
         url     = config['config']['url'] )
 
 def embed(video_link, vnf, image):
+    print(vnf)
     print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
     
     desc    = re.sub(r' http.*t\.co\S+', '', vnf['description'])
@@ -465,7 +516,8 @@ def embed(video_link, vnf, image):
         url        = config['config']['url'], 
         urlDesc    = urlDesc, 
         urlUser    = urlUser, 
-        urlLink    = urlLink )
+        urlLink    = urlLink,
+        tweetLink  = vnf['tweet'] )
 
 def tweetType(tweet): # Are we dealing with a Video, Image, or Text tweet?
     if 'extended_entities' in tweet: