diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d58716 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ + __pycache__/ \ No newline at end of file diff --git a/example.gif b/example.gif new file mode 100644 index 0000000..d73c757 Binary files /dev/null and b/example.gif differ diff --git a/readme.md b/readme.md index 27e8101..92e321d 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # TwitFix -very basic flask server that fixes twitter embeds in discord by using youtube-dl to grab the direct link to the MP4 file and embeds the link to it in a custom page +very basic flask server that fixes twitter embeds in discord by using youtube-dl to grab the direct link to the MP4 file and embeds the link to it in a custom page, with link caching This does work! but I'm new to flask, so it can probably be improved a great deal. @@ -9,16 +9,23 @@ This does work! but I'm new to flask, so it can probably be improved a great dea just put the url to the server, and directly after, the full URL to the tweet you want to embed **I now have a copy of this running on a Linode server, you can use it via the following url** -`http://twtfx.me/` + +https://fxtwitter.com/ + +You can also simply type out 'fx' directly before 'twitter.com' in any valid twitter video url, and that will convert it into a working TwitFix url, for example: + +![example](example.gif) **Note**: If you enjoy this service, please considering donating via [Ko-Fi](https://ko-fi.com/robin_universe) to help cover server costs ## How to run (server side) -this script uses the youtube-dl python module, along with flask, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py` ( will need sudo if you leave it at port 80 ) +this script uses the youtube-dl python module, along with flask and pymongo, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py` ( will need sudo if you leave it at port 80 ) By default I have the port set to 80, just cause that's what was convenient for me, but it can easily be changed, either using an environment variable, or changing the bottom line of the script itself +I have included some files to give you a head start on setting this server up with uWSGI, though if you decide to use uWSGI I suggest you set up mongoDB link caching by going into the script and change `link_cache_system` from `json` to `"DB"`, and inserting you mongoDB address, as having many workers writing to the same json file doesn't really work + This project is licensed under the **Do What The Fuck You Want Public License** diff --git a/requirements.txt b/requirements.txt index c4aad5a..9ce54bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ Flask youtube_dl +pymongo diff --git a/twitfix.ini b/twitfix.ini new file mode 100644 index 0000000..9812e2e --- /dev/null +++ b/twitfix.ini @@ -0,0 +1,11 @@ +[uwsgi] +module = wsgi:app + +master = true +processes = 5 + +socket = 80 +chmod-socket = 660 +vacuum = true + +die-on-term = true \ No newline at end of file diff --git a/twitfix.py b/twitfix.py index 79f5f1a..d3af460 100644 --- a/twitfix.py +++ b/twitfix.py @@ -1,4 +1,5 @@ from flask import Flask, render_template +import pymongo import youtube_dl import json import re @@ -6,14 +7,20 @@ import re app = Flask(__name__) pathregex = re.compile("\\w{1,15}\\/status\\/\\d{19}") -link_cache = {} -f = open('links.json',) -link_cache = json.load(f) -f.close() +link_cache_system = "json" # Select your prefered link cache system, "db" for mongoDB or "json" for locally writing a json file ( not great if using multiple workers ) + +if link_cache_system == "json": + link_cache = {} + f = open('links.json',) + link_cache = json.load(f) + f.close() +elif link_cache_system == "db": + client = pymongo.MongoClient("PUT YOUR MONGODB URL HERE") + db = client.TwitFix @app.route('/') def default(): - return render_template('default.html', message="TwitFix is an attempt to fix twitter video embeds in discord click this link to be directed to my github page for the project!") + return render_template('default.html', message="TwitFix is an attempt to fix twitter video embeds in discord! created by Robin Universe :) 💖 ") @app.route('/') def twitfix(subpath): @@ -43,28 +50,65 @@ def info(subpath): return result def embedVideo(vidlink): # Return a render template from a video url - if vidlink in link_cache: - print("Link located in cache") - return render_template('index.html', vidurl=link_cache[vidlink]['url'], desc=link_cache[vidlink]['description'], pic=link_cache[vidlink]['thumbnail'], user=link_cache[vidlink]['uploader'], vidlink=vidlink) + if link_cache_system == "db": + collection = db.linkCache + dbresult = collection.find_one({'tweet': vidlink}) + if dbresult != None: + print("Link located in DB cache") + return render_template('index.html', vidurl=dbresult['url'], desc=dbresult['description'], pic=dbresult['thumbnail'], user=dbresult['uploader'], vidlink=vidlink) + else: + with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl: + try: + print("Link not in json cache, downloading and adding details to cache file") + result = ydl.extract_info(vidlink, download=False) + vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader']) + + try: + out = db.linkCache.insert_one(vnf) + print("Link added to DB cache") + except Exception: + print("Failed to add link to DB cache") + + return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink) + + except Exception: + print("Failed to download link") + return render_template('default.html', message="Failed to scan your link!") + + elif link_cache_system == "json": + if vidlink in link_cache: + print("Link located in json cache") + return render_template('index.html', vidurl=link_cache[vidlink]['url'], desc=link_cache[vidlink]['description'], pic=link_cache[vidlink]['thumbnail'], user=link_cache[vidlink]['uploader'], vidlink=vidlink) + else: + with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl: + try: + print("Link not in json cache, downloading and adding details to cache file") + result = ydl.extract_info(vidlink, download=False) + vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader']) + link_cache[vidlink] = vnf + + with open("links.json", "w") as outfile: + json.dump(link_cache, outfile, indent=4, sort_keys=True) + + except Exception: # Just to keep from 500s that are messy + print("Failed to download link") + return render_template('default.html', message="Failed to scan your link!") + + return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink) else: - with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl: - try: - print("Link not in cache, downloading and adding details to cache file") - result = ydl.extract_info(vidlink, download=False) - vnf = vidInfo(result['url'], result['description'], result['thumbnail'], result['uploader']) - link_cache[vidlink] = vnf + try: + with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl: + result = ydl.extract_info(subpath, download=False) + vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader']) + return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink) + except Exception: + print("Failed to download link") + return render_template('default.html', message="Failed to scan your link!") - with open("links.json", "w") as outfile: - json.dump(link_cache, outfile, indent=4, sort_keys=True) - except Exception: # Just to keep from 500s that are messy - print("Failed to download link") - return render_template('default.html', message="Failed to scan your link!") - - return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink) - -def vidInfo(url, desc="", thumb="", uploader=""): # Return a dict of video info with default values +def vidInfo(url, tweet="", desc="", thumb="", uploader=""): # Return a dict of video info with default values vnf = { + "tweet" :tweet, "url" :url, "description" :desc, "thumbnail" :thumb, @@ -73,4 +117,4 @@ def vidInfo(url, desc="", thumb="", uploader=""): # Return a dict of video info return vnf if __name__ == "__main__": - app.run(host='0.0.0.0', port=80) + app.run(host='0.0.0.0') diff --git a/twitfix.service b/twitfix.service new file mode 100644 index 0000000..91905ec --- /dev/null +++ b/twitfix.service @@ -0,0 +1,13 @@ +[Unit] +Description=Init file for twitfix uwsgi instance +After=network.target + +[Service] +User=robin +Group=robin +WorkingDirectory=/home/robin/twitfix +Environment="PATH=/home/robin/twitfix/twitfixenv/bin" +ExecStart=/home/robin/twitfix/twitfixenv/bin/uwsgi --ini twitfix.ini + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..ed31819 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,4 @@ +from twitfix import app + +if __name__ == "__main__": + app.run() \ No newline at end of file