DynamoDB support & Serverless config update
This commit is contained in:
Dylan
parent
dee62f812a
commit
32e5376b98
3 changed files with 128 additions and 24 deletions
|
|
@ -1,3 +1,5 @@
|
|||
boto3==1.23.0
|
||||
botocore==1.26.0
|
||||
certifi==2021.10.8
|
||||
charset-normalizer==2.0.12
|
||||
click==8.0.4
|
||||
|
|
@ -9,13 +11,16 @@ idna==3.3
|
|||
importlib-metadata==4.8.3
|
||||
itsdangerous==2.0.1
|
||||
Jinja2==3.0.3
|
||||
jmespath==0.10.0
|
||||
MarkupSafe==2.0.1
|
||||
pymongo==4.1.1
|
||||
python-dateutil==2.8.2
|
||||
requests==2.27.1
|
||||
s3transfer==0.5.2
|
||||
six==1.16.0
|
||||
twitter==1.19.3
|
||||
typing-extensions==4.1.1
|
||||
typing_extensions==4.1.1
|
||||
urllib3==1.26.9
|
||||
Werkzeug==2.0.3
|
||||
youtube-dl==2021.12.17
|
||||
zipp==3.6.0
|
||||
zipp==3.6.0
|
||||
|
|
@ -1,8 +1,36 @@
|
|||
service: twitfix
|
||||
service: vxTwitter
|
||||
|
||||
provider:
|
||||
name: aws
|
||||
runtime: python3.6
|
||||
stage: dev
|
||||
iamRoleStatements:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- dynamodb:Query
|
||||
- dynamodb:Scan
|
||||
- dynamodb:GetItem
|
||||
- dynamodb:PutItem
|
||||
- dynamodb:UpdateItem
|
||||
- dynamodb:DeleteItem
|
||||
Resource:
|
||||
- { "Fn::GetAtt": ["vxTwitterDynamoTable", "Arn" ] }
|
||||
environment:
|
||||
CACHE_TABLE: ${self:custom.tableName}
|
||||
RUNNING_SERVERLESS: 1
|
||||
VXTWITTER_LINK_CACHE: dynamodb
|
||||
VXTWITTER_DATABASE: none
|
||||
VXTWITTER_DATABASE_TABLE: none
|
||||
VXTWITTER_METHOD: youtube-dl
|
||||
VXTWITTER_COLOR: \#43B581
|
||||
VXTWITTER_APP_NAME: vxTwitter
|
||||
VXTWITTER_REPO: https://github.com/dylanpdx/BetterTwitFix
|
||||
VXTWITTER_URL: https://vxtwitter.com
|
||||
# Twitter API keys
|
||||
VXTWITTER_TWITTER_API_KEY: none
|
||||
VXTWITTER_TWITTER_API_SECRET: none
|
||||
VXTWITTER_TWITTER_ACCESS_TOKEN: none
|
||||
VXTWITTER_TWITTER_ACCESS_SECRET: none
|
||||
|
||||
package:
|
||||
patterns:
|
||||
|
|
@ -16,7 +44,7 @@ plugins:
|
|||
- serverless-plugin-include-dependencies
|
||||
|
||||
functions:
|
||||
webhook:
|
||||
vxTwitterApp:
|
||||
handler: wsgi_handler.handler
|
||||
url: true
|
||||
layers:
|
||||
|
|
@ -24,8 +52,28 @@ functions:
|
|||
|
||||
|
||||
custom:
|
||||
tableName: 'users-table-${self:provider.stage}'
|
||||
wsgi:
|
||||
app: twitfix.app
|
||||
pythonRequirements:
|
||||
layer: true
|
||||
dockerizePip: true
|
||||
dockerizePip: true
|
||||
|
||||
|
||||
resources:
|
||||
Resources:
|
||||
vxTwitterDynamoTable:
|
||||
Type: 'AWS::DynamoDB::Table'
|
||||
Properties:
|
||||
AttributeDefinitions:
|
||||
-
|
||||
AttributeName: tweet
|
||||
AttributeType: S
|
||||
KeySchema:
|
||||
-
|
||||
AttributeName: tweet
|
||||
KeyType: HASH
|
||||
ProvisionedThroughput:
|
||||
ReadCapacityUnits: 1
|
||||
WriteCapacityUnits: 1
|
||||
TableName: ${self:custom.tableName}
|
||||
89
twitfix.py
89
twitfix.py
|
|
@ -11,6 +11,7 @@ import os
|
|||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import date
|
||||
import boto3
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app)
|
||||
|
|
@ -33,26 +34,46 @@ generate_embed_user_agents = [
|
|||
|
||||
# Read config from config.json. If it does not exist, create new.
|
||||
if not os.path.exists("config.json"):
|
||||
with open("config.json", "w") as outfile:
|
||||
default_config = {
|
||||
"config":{
|
||||
"link_cache":"json",
|
||||
"database":"[url to mongo database goes here]",
|
||||
"table":"TwiFix",
|
||||
"method":"youtube-dl",
|
||||
"color":"#43B581",
|
||||
"appname": "vxTwitter",
|
||||
"repo": "https://github.com/dylanpdx/BetterTwitFix",
|
||||
"url": "https://vxtwitter.com"
|
||||
},
|
||||
"api":{"api_key":"[api_key goes here]",
|
||||
"api_secret":"[api_secret goes here]",
|
||||
"access_token":"[access_token goes here]",
|
||||
"access_secret":"[access_secret goes here]"
|
||||
serverless_check = os.environ.get('RUNNING_SERVERLESS')
|
||||
if serverless_check == None: # Running on local pc, therefore we can access the filesystem
|
||||
with open("config.json", "w") as outfile:
|
||||
default_config = {
|
||||
"config":{
|
||||
"link_cache":"dynamodb",
|
||||
"database":"[url to mongo database goes here]",
|
||||
"table":"TwiFix",
|
||||
"method":"youtube-dl",
|
||||
"color":"#43B581",
|
||||
"appname": "vxTwitter",
|
||||
"repo": "https://github.com/dylanpdx/BetterTwitFix",
|
||||
"url": "https://vxtwitter.com"
|
||||
},
|
||||
"api":{"api_key":"[api_key goes here]",
|
||||
"api_secret":"[api_secret goes here]",
|
||||
"access_token":"[access_token goes here]",
|
||||
"access_secret":"[access_secret goes here]"
|
||||
}
|
||||
}
|
||||
json.dump(default_config, outfile, indent=4, sort_keys=True)
|
||||
else: # Running on serverless, therefore we cannot access the filesystem and must use environment variables
|
||||
default_config = {
|
||||
"config":{
|
||||
"link_cache":os.environ['VXTWITTER_LINK_CACHE'],
|
||||
"database":os.environ['VXTWITTER_DATABASE'],
|
||||
"table":os.environ['VXTWITTER_DATABASE_TABLE'],
|
||||
"method":os.environ['VXTWITTER_METHOD'],
|
||||
"color":os.environ['VXTWITTER_COLOR'],
|
||||
"appname": os.environ['VXTWITTER_APP_NAME'],
|
||||
"repo": os.environ['VXTWITTER_REPO'],
|
||||
"url": os.environ['VXTWITTER_URL'],
|
||||
},
|
||||
"api":{
|
||||
"api_key":os.environ['VXTWITTER_TWITTER_API_KEY'],
|
||||
"api_secret":os.environ['VXTWITTER_TWITTER_API_SECRET'],
|
||||
"access_token":os.environ['VXTWITTER_TWITTER_ACCESS_TOKEN'],
|
||||
"access_secret":os.environ['VXTWITTER_TWITTER_ACCESS_SECRET']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
json.dump(default_config, outfile, indent=4, sort_keys=True)
|
||||
|
||||
config = default_config
|
||||
else:
|
||||
|
|
@ -66,6 +87,9 @@ if config['config']['method'] in ('api', 'hybrid'):
|
|||
twitter_api = twitter.Twitter(auth=auth)
|
||||
|
||||
link_cache_system = config['config']['link_cache']
|
||||
DYNAMO_CACHE_TBL=None
|
||||
if link_cache_system=="dynamodb":
|
||||
DYNAMO_CACHE_TBL=os.environ['CACHE_TABLE']
|
||||
|
||||
if link_cache_system == "json":
|
||||
link_cache = {}
|
||||
|
|
@ -81,6 +105,8 @@ elif link_cache_system == "db":
|
|||
client = pymongo.MongoClient(config['config']['database'], connect=False)
|
||||
table = config['config']['table']
|
||||
db = client[table]
|
||||
elif link_cache_system == "dynamodb":
|
||||
client = boto3.resource('dynamodb')
|
||||
|
||||
@app.route('/') # If the useragent is discord, return the embed, if not, redirect to configured repo directly
|
||||
def default():
|
||||
|
|
@ -381,6 +407,20 @@ def getVnfFromLinkCache(video_link):
|
|||
else:
|
||||
print(" ➤ [ X ] Link not in json cache")
|
||||
return None
|
||||
elif link_cache_system == "dynamodb":
|
||||
table = client.Table(DYNAMO_CACHE_TBL)
|
||||
response = table.get_item(
|
||||
Key={
|
||||
'tweet': video_link
|
||||
}
|
||||
)
|
||||
if 'Item' in response:
|
||||
print("Link located in dynamodb cache")
|
||||
vnf = response['Item']
|
||||
return vnf
|
||||
else:
|
||||
print(" ➤ [ X ] Link not in dynamodb cache")
|
||||
return None
|
||||
|
||||
def addVnfToLinkCache(video_link, vnf):
|
||||
if link_cache_system == "db":
|
||||
|
|
@ -396,6 +436,16 @@ def addVnfToLinkCache(video_link, vnf):
|
|||
with open("links.json", "w") as outfile:
|
||||
json.dump(link_cache, outfile, indent=4, sort_keys=True)
|
||||
return None
|
||||
elif link_cache_system == "dynamodb":
|
||||
table = client.Table(DYNAMO_CACHE_TBL)
|
||||
table.put_item(
|
||||
Item={
|
||||
'tweet': video_link,
|
||||
'vnf': vnf
|
||||
}
|
||||
)
|
||||
print(" ➤ [ + ] Link added to dynamodb cache ")
|
||||
return True
|
||||
|
||||
def message(text):
|
||||
return render_template(
|
||||
|
|
@ -407,6 +457,7 @@ def message(text):
|
|||
url = config['config']['url'] )
|
||||
|
||||
def embed(video_link, vnf, image):
|
||||
print(vnf)
|
||||
print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
|
||||
|
||||
desc = re.sub(r' http.*t\.co\S+', '', vnf['description'])
|
||||
|
|
|
|||
Loading…
Reference in a new issue