Fixed DynamoDB integration

DynamoDB support & Serverless config update
Updated serverless.yml to dockerize pip packages
2022-05-18 18:21:00 +01:00 · 2022-05-16 19:28:31 +01:00 · 2022-05-15 17:06:42 +01:00 · 2022-05-15 14:30:18 +01:00 · 2022-05-15 01:47:20 +01:00 · 2022-05-15 01:14:41 +01:00
10 changed files with 11025 additions and 113 deletions
--- a/.gitignore
+++ b/.gitignore
@ -2,3 +2,9 @@ config.json
 __pycache__
 venv/
 links.json
+node_modules
+#SERVERLESS STUFF
+admin.env
+.env
+_meta
+.serverless
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@ -0,0 +1,28 @@
+{
+  "name": "bettertwitfix",
+  "version": "1.0.0",
+  "description": "(A fork of TwitFix)\r Basic flask server that serves fixed twitter video embeds to desktop discord by using either the Twitter API or Youtube-DL to grab tweet video information. This also automatically embeds the first link in the text of non video tweets (API Only)",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dylanpdx/BetterTwitFix.git"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/dylanpdx/BetterTwitFix/issues"
+  },
+  "homepage": "https://github.com/dylanpdx/BetterTwitFix#readme",
+  "dependencies": {
+    "serverless-wsgi": "^3.0.0"
+  },
+  "devDependencies": {
+    "serverless-plugin-common-excludes": "^4.0.0",
+    "serverless-plugin-include-dependencies": "^5.0.0",
+    "serverless-python-requirements": "^5.4.0"
+  }
+}
--- a/requirements.txt
+++ b/requirements.txt
--- a/serverless.yml
+++ b/serverless.yml
@ -0,0 +1,79 @@
+service: vxTwitter
+
+provider:
+  name: aws
+  runtime: python3.6
+  stage: dev
+  iamRoleStatements:
+    - Effect: Allow
+      Action:
+        - dynamodb:Query
+        - dynamodb:Scan
+        - dynamodb:GetItem
+        - dynamodb:PutItem
+        - dynamodb:UpdateItem
+        - dynamodb:DeleteItem
+      Resource:
+        - { "Fn::GetAtt": ["vxTwitterDynamoTable", "Arn" ] }
+  environment:
+    CACHE_TABLE: ${self:custom.tableName}
+    RUNNING_SERVERLESS: 1
+    VXTWITTER_LINK_CACHE: dynamodb
+    VXTWITTER_DATABASE: none
+    VXTWITTER_DATABASE_TABLE: none
+    VXTWITTER_METHOD: youtube-dl
+    VXTWITTER_COLOR: \#43B581
+    VXTWITTER_APP_NAME: vxTwitter
+    VXTWITTER_REPO: https://github.com/dylanpdx/BetterTwitFix
+    VXTWITTER_URL: https://vxtwitter.com
+    # Twitter API keys
+    VXTWITTER_TWITTER_API_KEY: none
+    VXTWITTER_TWITTER_API_SECRET: none
+    VXTWITTER_TWITTER_ACCESS_TOKEN: none
+    VXTWITTER_TWITTER_ACCESS_SECRET: none
+
+package:
+  patterns:
+    - '!node_modules/**'
+    - '!venv/**'
+
+plugins:
+  - serverless-wsgi
+  - serverless-python-requirements
+  - serverless-plugin-common-excludes
+  - serverless-plugin-include-dependencies
+
+functions:
+  vxTwitterApp:
+    handler: wsgi_handler.handler
+    url: true
+    layers:
+      - Ref: PythonRequirementsLambdaLayer
+
+
+custom:
+  tableName: 'users-table-${self:provider.stage}'
+  wsgi:
+    app: twitfix.app
+  pythonRequirements:
+    layer: true
+    dockerizePip: true
+
+	
+resources:
+  Resources:
+    vxTwitterDynamoTable:
+      Type: 'AWS::DynamoDB::Table'
+      Properties:
+        AttributeDefinitions:
+          -
+            AttributeName: tweet
+            AttributeType: S
+        KeySchema:
+          -
+            AttributeName: tweet
+            KeyType: HASH
+        ProvisionedThroughput:
+          ReadCapacityUnits: 1
+          WriteCapacityUnits: 1
+        TableName: ${self:custom.tableName}
--- a/templates/image.html
+++ b/templates/image.html
@ -14,22 +14,17 @@
    {{ pfp }}     - URL of the posters PFP
 -->
 {% block head %}
-    <meta content="text/html; charset=UTF-8"        http-equiv="Content-Type" />
-    <meta content="{{ color }}"                     name="theme-color" />
-    <meta property="og:site_name"                   content="{{ appname }}"/>
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}" />

-    <meta name="twitter:card"                       content="summary_large_image" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
-    <meta name="twitter:creator"                    content="@{{ user }}" />
+<meta name="twitter:card" content="summary_large_image" />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
+<meta name="twitter:image" content="{{ pic }}" />
+<meta name="twitter:image" content="{{ pic }}" />
+<meta name="twitter:creator" content="@{{ user }}" />

-    <meta property="og:description"                 content="{{ desc }}"  />
+<meta property="og:description" content="{{ desc }}" />

-    <!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}
+<!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
--- a/templates/index.html
+++ b/templates/index.html
@ -1,37 +1,30 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}">

-{% block head %}
-    <meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
-    <meta content="{{ color }}" name="theme-color" />
-    <meta property="og:site_name" content="{{ appname }}">
-
-    <meta name="twitter:card"                       content="player" />
-    <meta name="twitter:title"                      content="{{ user }}" />
-    <meta name="twitter:image"                      content="{{ pic }}" />
-    <!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
+<meta name="twitter:card" content="player" />
+<meta name="twitter:title" content="{{ user }}" />
+<meta name="twitter:image" content="{{ pic }}" />
+<!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
    <meta name="twitter:player"                     content="{{ vidurl }}" />
    <meta name="twitter:description"                content="{{ desc }}" />!-->
-    <meta name="twitter:player:width"               content="720" />
-    <meta name="twitter:player:height"              content="480" />
-    <meta name="twitter:player:stream"              content="{{ vidurl }}" />
-    <meta name="twitter:player:stream:content_type" content="video/mp4" /> 
+<meta name="twitter:player:width" content="720" />
+<meta name="twitter:player:height" content="480" />
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
+<meta name="twitter:player:stream:content_type" content="video/mp4" />


-    <meta property="og:url"                content="{{ vidlink }}" />
-    <meta property="og:video"              content="{{ vidurl }}" />
-    <meta property="og:video:secure_url"   content="{{ vidurl }}" />
-    <meta property="og:video:type"         content="video/mp4" />
-    <meta property="og:video:width"        content="720" />
-    <meta property="og:video:height"       content="480" />
-    <meta property="og:title"              content="{{ user }}" />
-    <!--<meta property="og:description"        content="{{ desc }}" />!-->
-    <meta property="og:image"              content="{{ pic }}" />
+<meta property="og:url" content="{{ vidlink }}" />
+<meta property="og:video" content="{{ vidurl }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
+<meta property="og:video:type" content="video/mp4" />
+<meta property="og:video:width" content="720" />
+<meta property="og:video:height" content="480" />
+<meta property="og:title" content="{{ user }}" />
+<!--<meta property="og:description"        content="{{ desc }}" />!-->
+<meta property="og:image" content="{{ pic }}" />

-        <!--!-->
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}
+<!--!-->
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
--- a/templates/text.html
+++ b/templates/text.html
@ -18,22 +18,17 @@
    {{ time }}    - Time Created
 -->
 {% block head %}
-    <meta content="text/html; charset=UTF-8"        http-equiv="Content-Type" />
-    <meta content="{{ color }}"                     name="theme-color" />
-    <meta property="og:site_name"                   content="{{ appname }}"/>
-    <meta property="og:image"                       content="{{ pfp }}"   />
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}" />
+<meta property="og:image" content="{{ pfp }}" />

-    <meta name="twitter:card"                       content="tweet"       />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
-    <meta name="twitter:creator"                    content="@{{ user }}" />
+<meta name="twitter:card" content="tweet" />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
+<meta name="twitter:image" content="{{ pic }}" />
+<meta name="twitter:creator" content="@{{ user }}" />

-    <meta property="og:description"                 content="{{ desc }}"  />
+<meta property="og:description" content="{{ desc }}" />

-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
--- a/templates/video.html
+++ b/templates/video.html
@ -1,31 +1,24 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}">

-{% block head %}
-    <meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
-    <meta content="{{ color }}"  name="theme-color" />
-    <meta property="og:site_name" content="{{ appname }}">
+<meta name="twitter:card" content="player" />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
+<meta name="twitter:image" content="{{ pic }}" />
+<meta name="twitter:player:width" content="720" />
+<meta name="twitter:player:height" content="480" />
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
+<meta name="twitter:player:stream:content_type" content="video/mp4" />

-    <meta name="twitter:card"                       content="player" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
-    <meta name="twitter:image"                      content="{{ pic }}" />
-    <meta name="twitter:player:width"               content="720" />
-    <meta name="twitter:player:height"              content="480" />
-    <meta name="twitter:player:stream"              content="{{ vidurl }}" />
-    <meta name="twitter:player:stream:content_type" content="video/mp4" /> 
+<meta property="og:url" content="{{ vidlink }}" />
+<meta property="og:video" content="{{ vidurl }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
+<meta property="og:video:type" content="video/mp4" />
+<meta property="og:video:width" content="720" />
+<meta property="og:video:height" content="480" />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
+<meta property="og:image" content="{{ pic }}" />

-    <meta property="og:url"                content="{{ vidlink }}" />
-    <meta property="og:video"              content="{{ vidurl }}" />
-    <meta property="og:video:secure_url"   content="{{ vidurl }}" />
-    <meta property="og:video:type"         content="video/mp4" />
-    <meta property="og:video:width"        content="720" />
-    <meta property="og:video:height"       content="480" />
-    <meta name="twitter:title"             content="{{ user }} (@{{ screenName }})"  />
-    <meta property="og:image"              content="{{ pic }}" />
-
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
--- a/twitfix.py
+++ b/twitfix.py
@ -11,6 +11,7 @@ import os
 import urllib.parse
 import urllib.request
 from datetime import date
+import boto3

 app = Flask(__name__)
 CORS(app)
@ -33,10 +34,12 @@ generate_embed_user_agents = [

 # Read config from config.json. If it does not exist, create new.
 if not os.path.exists("config.json"):
+    serverless_check = os.environ.get('RUNNING_SERVERLESS')
+    if serverless_check == None: # Running on local pc, therefore we can access the filesystem
        with open("config.json", "w") as outfile:
            default_config = {
                "config":{
-                "link_cache":"json",
+                    "link_cache":"dynamodb",
                    "database":"[url to mongo database goes here]",
                    "table":"TwiFix",
                    "method":"youtube-dl", 
@ -51,8 +54,26 @@ if not os.path.exists("config.json"):
                "access_secret":"[access_secret goes here]"
                }
            }
-
            json.dump(default_config, outfile, indent=4, sort_keys=True)
+    else: # Running on serverless, therefore we cannot access the filesystem and must use environment variables
+            default_config = {
+                "config":{
+                    "link_cache":os.environ['VXTWITTER_LINK_CACHE'],
+                    "database":os.environ['VXTWITTER_DATABASE'],
+                    "table":os.environ['VXTWITTER_DATABASE_TABLE'],
+                    "method":os.environ['VXTWITTER_METHOD'],
+                    "color":os.environ['VXTWITTER_COLOR'],
+                    "appname": os.environ['VXTWITTER_APP_NAME'],
+                    "repo": os.environ['VXTWITTER_REPO'],
+                    "url": os.environ['VXTWITTER_URL'],
+                    },
+                "api":{
+                    "api_key":os.environ['VXTWITTER_TWITTER_API_KEY'],
+                    "api_secret":os.environ['VXTWITTER_TWITTER_API_SECRET'],
+                    "access_token":os.environ['VXTWITTER_TWITTER_ACCESS_TOKEN'],
+                    "access_secret":os.environ['VXTWITTER_TWITTER_ACCESS_SECRET']
+                }
+            }

    config = default_config
 else:
@ -66,6 +87,9 @@ if config['config']['method'] in ('api', 'hybrid'):
    twitter_api = twitter.Twitter(auth=auth)

 link_cache_system = config['config']['link_cache']
+DYNAMO_CACHE_TBL=None
+if link_cache_system=="dynamodb":
+    DYNAMO_CACHE_TBL=os.environ['CACHE_TABLE']

 if link_cache_system == "json":
    link_cache = {}
@ -81,6 +105,8 @@ elif link_cache_system == "db":
    client = pymongo.MongoClient(config['config']['database'], connect=False)
    table = config['config']['table']
    db = client[table]
+elif link_cache_system == "dynamodb":
+    client = boto3.resource('dynamodb')

@app.route('/') # If the useragent is discord, return the embed, if not, redirect to configured repo directly
 def default():
@ -266,7 +292,7 @@ def link_to_vnf_from_api(video_link):
    twid = int(re.sub(r'\?.*$','',video_link.rsplit("/", 1)[-1])) # gets the tweet ID as a int from the passed url
    tweet = twitter_api.statuses.show(_id=twid, tweet_mode="extended")
    # For when I need to poke around and see what a tweet looks like
-    #print(tweet)
+    print(tweet)
    imgs = ["","","","", ""]
    print(" ➤ [ + ] Tweet Type: " + tweetType(tweet))
    # Check to see if tweet has a video, if not, make the url passed to the VNF the first t.co link in the tweet
@ -381,6 +407,20 @@ def getVnfFromLinkCache(video_link):
        else:
            print(" ➤ [ X ] Link not in json cache")
            return None
+    elif link_cache_system == "dynamodb":
+        table = client.Table(DYNAMO_CACHE_TBL)
+        response = table.get_item(
+            Key={
+                'tweet': video_link
+            }
+        )
+        if 'Item' in response:
+            print("Link located in dynamodb cache")
+            vnf = response['Item']['vnf']
+            return vnf
+        else:
+            print(" ➤ [ X ] Link not in dynamodb cache")
+            return None

 def addVnfToLinkCache(video_link, vnf):
    if link_cache_system == "db":
@ -396,6 +436,16 @@ def addVnfToLinkCache(video_link, vnf):
        with open("links.json", "w") as outfile: 
            json.dump(link_cache, outfile, indent=4, sort_keys=True)
            return None
+    elif link_cache_system == "dynamodb":
+        table = client.Table(DYNAMO_CACHE_TBL)
+        table.put_item(
+            Item={
+                'tweet': video_link,
+                'vnf': vnf
+            }
+        )
+        print(" ➤ [ + ] Link added to dynamodb cache ")
+        return True

 def message(text):
    return render_template(
@ -407,6 +457,7 @@ def message(text):
        url     = config['config']['url'] )

 def embed(video_link, vnf, image):
+    print(vnf)
    print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
    
    desc    = re.sub(r' http.*t\.co\S+', '', vnf['description'])
@ -465,7 +516,8 @@ def embed(video_link, vnf, image):
        url        = config['config']['url'], 
        urlDesc    = urlDesc, 
        urlUser    = urlUser, 
-        urlLink    = urlLink )
+        urlLink    = urlLink,
+        tweetLink  = vnf['tweet'] )

 def tweetType(tweet): # Are we dealing with a Video, Image, or Text tweet?
    if 'extended_entities' in tweet:
Author	SHA1	Message	Date
Dylan	f2eefbe5a5	Fixed DynamoDB integration	2022-05-18 18:21:00 +01:00
Dylan	32e5376b98	DynamoDB support & Serverless config update	2022-05-16 19:28:31 +01:00
Dylan	dee62f812a	Updated serverless.yml to dockerize pip packages	2022-05-15 17:06:42 +01:00
Dylan	43d69a1d87	serverless wsgi files	2022-05-15 14:30:18 +01:00
Dylan	96246aa921	Squashed commit of the following: commit `fba1256efd` Author: Dylan <dylanpdx@gmail.com> Date: Sun May 15 01:42:22 2022 +0100 Fixed infinite redirect(?) commit `85c682ab11` Author: Dylan <dylanpdx@gmail.com> Date: Sun May 15 01:15:15 2022 +0100 Updated gitignore	2022-05-15 01:47:20 +01:00
Dylan	2c9563fafe	Serverless config	2022-05-15 01:14:41 +01:00