Compare commits

...

6 commits

Author SHA1 Message Date
Dylan
f2eefbe5a5 Fixed DynamoDB integration 2022-05-18 18:21:00 +01:00
Dylan
32e5376b98 DynamoDB support & Serverless config update 2022-05-16 19:28:31 +01:00
Dylan
dee62f812a Updated serverless.yml to dockerize pip packages 2022-05-15 17:06:42 +01:00
Dylan
43d69a1d87 serverless wsgi files 2022-05-15 14:30:18 +01:00
Dylan
96246aa921 Squashed commit of the following:
commit fba1256efd
Author: Dylan <dylanpdx@gmail.com>
Date:   Sun May 15 01:42:22 2022 +0100

    Fixed infinite redirect(?)

commit 85c682ab11
Author: Dylan <dylanpdx@gmail.com>
Date:   Sun May 15 01:15:15 2022 +0100

    Updated gitignore
2022-05-15 01:47:20 +01:00
Dylan
2c9563fafe Serverless config 2022-05-15 01:14:41 +01:00
10 changed files with 11025 additions and 113 deletions

8
.gitignore vendored
View file

@ -1,4 +1,10 @@
config.json
__pycache__
venv/
links.json
links.json
node_modules
#SERVERLESS STUFF
admin.env
.env
_meta
.serverless

10771
package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

28
package.json Normal file
View file

@ -0,0 +1,28 @@
{
"name": "bettertwitfix",
"version": "1.0.0",
"description": "(A fork of TwitFix)\r Basic flask server that serves fixed twitter video embeds to desktop discord by using either the Twitter API or Youtube-DL to grab tweet video information. This also automatically embeds the first link in the text of non video tweets (API Only)",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git+https://github.com/dylanpdx/BetterTwitFix.git"
},
"keywords": [],
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/dylanpdx/BetterTwitFix/issues"
},
"homepage": "https://github.com/dylanpdx/BetterTwitFix#readme",
"dependencies": {
"serverless-wsgi": "^3.0.0"
},
"devDependencies": {
"serverless-plugin-common-excludes": "^4.0.0",
"serverless-plugin-include-dependencies": "^5.0.0",
"serverless-python-requirements": "^5.4.0"
}
}

Binary file not shown.

79
serverless.yml Normal file
View file

@ -0,0 +1,79 @@
service: vxTwitter
provider:
name: aws
runtime: python3.6
stage: dev
iamRoleStatements:
- Effect: Allow
Action:
- dynamodb:Query
- dynamodb:Scan
- dynamodb:GetItem
- dynamodb:PutItem
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource:
- { "Fn::GetAtt": ["vxTwitterDynamoTable", "Arn" ] }
environment:
CACHE_TABLE: ${self:custom.tableName}
RUNNING_SERVERLESS: 1
VXTWITTER_LINK_CACHE: dynamodb
VXTWITTER_DATABASE: none
VXTWITTER_DATABASE_TABLE: none
VXTWITTER_METHOD: youtube-dl
VXTWITTER_COLOR: \#43B581
VXTWITTER_APP_NAME: vxTwitter
VXTWITTER_REPO: https://github.com/dylanpdx/BetterTwitFix
VXTWITTER_URL: https://vxtwitter.com
# Twitter API keys
VXTWITTER_TWITTER_API_KEY: none
VXTWITTER_TWITTER_API_SECRET: none
VXTWITTER_TWITTER_ACCESS_TOKEN: none
VXTWITTER_TWITTER_ACCESS_SECRET: none
package:
patterns:
- '!node_modules/**'
- '!venv/**'
plugins:
- serverless-wsgi
- serverless-python-requirements
- serverless-plugin-common-excludes
- serverless-plugin-include-dependencies
functions:
vxTwitterApp:
handler: wsgi_handler.handler
url: true
layers:
- Ref: PythonRequirementsLambdaLayer
custom:
tableName: 'users-table-${self:provider.stage}'
wsgi:
app: twitfix.app
pythonRequirements:
layer: true
dockerizePip: true
resources:
Resources:
vxTwitterDynamoTable:
Type: 'AWS::DynamoDB::Table'
Properties:
AttributeDefinitions:
-
AttributeName: tweet
AttributeType: S
KeySchema:
-
AttributeName: tweet
KeyType: HASH
ProvisionedThroughput:
ReadCapacityUnits: 1
WriteCapacityUnits: 1
TableName: ${self:custom.tableName}

View file

@ -14,22 +14,17 @@
{{ pfp }} - URL of the posters PFP
-->
{% block head %}
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}"/>
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:creator" content="@{{ user }}" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:creator" content="@{{ user }}" />
<meta property="og:description" content="{{ desc }}" />
<!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
<meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
{% endblock %}
<meta property="og:description" content="{{ desc }}" />
{% block body %}
Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
{% endblock %}
<!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

View file

@ -1,37 +1,30 @@
{% extends 'base.html' %}
{% extends 'base.html' %} {% block head %}
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}">
{% block head %}
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}">
<meta name="twitter:card" content="player" />
<meta name="twitter:title" content="{{ user }}" />
<meta name="twitter:image" content="{{ pic }}" />
<!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
<meta name="twitter:card" content="player" />
<meta name="twitter:title" content="{{ user }}" />
<meta name="twitter:image" content="{{ pic }}" />
<!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
<meta name="twitter:player" content="{{ vidurl }}" />
<meta name="twitter:description" content="{{ desc }}" />!-->
<meta name="twitter:player:width" content="720" />
<meta name="twitter:player:height" content="480" />
<meta name="twitter:player:stream" content="{{ vidurl }}" />
<meta name="twitter:player:stream:content_type" content="video/mp4" />
<meta name="twitter:player:width" content="720" />
<meta name="twitter:player:height" content="480" />
<meta name="twitter:player:stream" content="{{ vidurl }}" />
<meta name="twitter:player:stream:content_type" content="video/mp4" />
<meta property="og:url" content="{{ vidlink }}" />
<meta property="og:video" content="{{ vidurl }}" />
<meta property="og:video:secure_url" content="{{ vidurl }}" />
<meta property="og:video:type" content="video/mp4" />
<meta property="og:video:width" content="720" />
<meta property="og:video:height" content="480" />
<meta property="og:title" content="{{ user }}" />
<!--<meta property="og:description" content="{{ desc }}" />!-->
<meta property="og:image" content="{{ pic }}" />
<!--!-->
<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
<meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
{% endblock %}
<meta property="og:url" content="{{ vidlink }}" />
<meta property="og:video" content="{{ vidurl }}" />
<meta property="og:video:secure_url" content="{{ vidurl }}" />
<meta property="og:video:type" content="video/mp4" />
<meta property="og:video:width" content="720" />
<meta property="og:video:height" content="480" />
<meta property="og:title" content="{{ user }}" />
<!--<meta property="og:description" content="{{ desc }}" />!-->
<meta property="og:image" content="{{ pic }}" />
{% block body %}
Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
{% endblock %}
<!--!-->
<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

View file

@ -18,22 +18,17 @@
{{ time }} - Time Created
-->
{% block head %}
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}"/>
<meta property="og:image" content="{{ pfp }}" />
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}" />
<meta property="og:image" content="{{ pfp }}" />
<meta name="twitter:card" content="tweet" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:creator" content="@{{ user }}" />
<meta name="twitter:card" content="tweet" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:creator" content="@{{ user }}" />
<meta property="og:description" content="{{ desc }}" />
<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
<meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
{% endblock %}
<meta property="og:description" content="{{ desc }}" />
{% block body %}
Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
{% endblock %}
<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

View file

@ -1,31 +1,24 @@
{% extends 'base.html' %}
{% extends 'base.html' %} {% block head %}
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}">
{% block head %}
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
<meta content="{{ color }}" name="theme-color" />
<meta property="og:site_name" content="{{ appname }}">
<meta name="twitter:card" content="player" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:player:width" content="720" />
<meta name="twitter:player:height" content="480" />
<meta name="twitter:player:stream" content="{{ vidurl }}" />
<meta name="twitter:player:stream:content_type" content="video/mp4" />
<meta name="twitter:card" content="player" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta name="twitter:image" content="{{ pic }}" />
<meta name="twitter:player:width" content="720" />
<meta name="twitter:player:height" content="480" />
<meta name="twitter:player:stream" content="{{ vidurl }}" />
<meta name="twitter:player:stream:content_type" content="video/mp4" />
<meta property="og:url" content="{{ vidlink }}" />
<meta property="og:video" content="{{ vidurl }}" />
<meta property="og:video:secure_url" content="{{ vidurl }}" />
<meta property="og:video:type" content="video/mp4" />
<meta property="og:video:width" content="720" />
<meta property="og:video:height" content="480" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta property="og:image" content="{{ pic }}" />
<meta property="og:url" content="{{ vidlink }}" />
<meta property="og:video" content="{{ vidurl }}" />
<meta property="og:video:secure_url" content="{{ vidurl }}" />
<meta property="og:video:type" content="video/mp4" />
<meta property="og:video:width" content="720" />
<meta property="og:video:height" content="480" />
<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
<meta property="og:image" content="{{ pic }}" />
<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
<meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
{% endblock %}
{% block body %}
Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
{% endblock %}
<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

View file

@ -11,6 +11,7 @@ import os
import urllib.parse
import urllib.request
from datetime import date
import boto3
app = Flask(__name__)
CORS(app)
@ -33,26 +34,46 @@ generate_embed_user_agents = [
# Read config from config.json. If it does not exist, create new.
if not os.path.exists("config.json"):
with open("config.json", "w") as outfile:
default_config = {
"config":{
"link_cache":"json",
"database":"[url to mongo database goes here]",
"table":"TwiFix",
"method":"youtube-dl",
"color":"#43B581",
"appname": "vxTwitter",
"repo": "https://github.com/dylanpdx/BetterTwitFix",
"url": "https://vxtwitter.com"
},
"api":{"api_key":"[api_key goes here]",
"api_secret":"[api_secret goes here]",
"access_token":"[access_token goes here]",
"access_secret":"[access_secret goes here]"
serverless_check = os.environ.get('RUNNING_SERVERLESS')
if serverless_check == None: # Running on local pc, therefore we can access the filesystem
with open("config.json", "w") as outfile:
default_config = {
"config":{
"link_cache":"dynamodb",
"database":"[url to mongo database goes here]",
"table":"TwiFix",
"method":"youtube-dl",
"color":"#43B581",
"appname": "vxTwitter",
"repo": "https://github.com/dylanpdx/BetterTwitFix",
"url": "https://vxtwitter.com"
},
"api":{"api_key":"[api_key goes here]",
"api_secret":"[api_secret goes here]",
"access_token":"[access_token goes here]",
"access_secret":"[access_secret goes here]"
}
}
json.dump(default_config, outfile, indent=4, sort_keys=True)
else: # Running on serverless, therefore we cannot access the filesystem and must use environment variables
default_config = {
"config":{
"link_cache":os.environ['VXTWITTER_LINK_CACHE'],
"database":os.environ['VXTWITTER_DATABASE'],
"table":os.environ['VXTWITTER_DATABASE_TABLE'],
"method":os.environ['VXTWITTER_METHOD'],
"color":os.environ['VXTWITTER_COLOR'],
"appname": os.environ['VXTWITTER_APP_NAME'],
"repo": os.environ['VXTWITTER_REPO'],
"url": os.environ['VXTWITTER_URL'],
},
"api":{
"api_key":os.environ['VXTWITTER_TWITTER_API_KEY'],
"api_secret":os.environ['VXTWITTER_TWITTER_API_SECRET'],
"access_token":os.environ['VXTWITTER_TWITTER_ACCESS_TOKEN'],
"access_secret":os.environ['VXTWITTER_TWITTER_ACCESS_SECRET']
}
}
}
json.dump(default_config, outfile, indent=4, sort_keys=True)
config = default_config
else:
@ -66,6 +87,9 @@ if config['config']['method'] in ('api', 'hybrid'):
twitter_api = twitter.Twitter(auth=auth)
link_cache_system = config['config']['link_cache']
DYNAMO_CACHE_TBL=None
if link_cache_system=="dynamodb":
DYNAMO_CACHE_TBL=os.environ['CACHE_TABLE']
if link_cache_system == "json":
link_cache = {}
@ -81,6 +105,8 @@ elif link_cache_system == "db":
client = pymongo.MongoClient(config['config']['database'], connect=False)
table = config['config']['table']
db = client[table]
elif link_cache_system == "dynamodb":
client = boto3.resource('dynamodb')
@app.route('/') # If the useragent is discord, return the embed, if not, redirect to configured repo directly
def default():
@ -266,7 +292,7 @@ def link_to_vnf_from_api(video_link):
twid = int(re.sub(r'\?.*$','',video_link.rsplit("/", 1)[-1])) # gets the tweet ID as a int from the passed url
tweet = twitter_api.statuses.show(_id=twid, tweet_mode="extended")
# For when I need to poke around and see what a tweet looks like
#print(tweet)
print(tweet)
imgs = ["","","","", ""]
print(" ➤ [ + ] Tweet Type: " + tweetType(tweet))
# Check to see if tweet has a video, if not, make the url passed to the VNF the first t.co link in the tweet
@ -381,6 +407,20 @@ def getVnfFromLinkCache(video_link):
else:
print(" ➤ [ X ] Link not in json cache")
return None
elif link_cache_system == "dynamodb":
table = client.Table(DYNAMO_CACHE_TBL)
response = table.get_item(
Key={
'tweet': video_link
}
)
if 'Item' in response:
print("Link located in dynamodb cache")
vnf = response['Item']['vnf']
return vnf
else:
print(" ➤ [ X ] Link not in dynamodb cache")
return None
def addVnfToLinkCache(video_link, vnf):
if link_cache_system == "db":
@ -396,6 +436,16 @@ def addVnfToLinkCache(video_link, vnf):
with open("links.json", "w") as outfile:
json.dump(link_cache, outfile, indent=4, sort_keys=True)
return None
elif link_cache_system == "dynamodb":
table = client.Table(DYNAMO_CACHE_TBL)
table.put_item(
Item={
'tweet': video_link,
'vnf': vnf
}
)
print(" ➤ [ + ] Link added to dynamodb cache ")
return True
def message(text):
return render_template(
@ -407,6 +457,7 @@ def message(text):
url = config['config']['url'] )
def embed(video_link, vnf, image):
print(vnf)
print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
desc = re.sub(r' http.*t\.co\S+', '', vnf['description'])
@ -465,7 +516,8 @@ def embed(video_link, vnf, image):
url = config['config']['url'],
urlDesc = urlDesc,
urlUser = urlUser,
urlLink = urlLink )
urlLink = urlLink,
tweetLink = vnf['tweet'] )
def tweetType(tweet): # Are we dealing with a Video, Image, or Text tweet?
if 'extended_entities' in tweet: