Added a new link caching system, updated the readme, started ignoreing pycache

.gitignore

@@ -0,0 +1 @@
+ __pycache__/

readme.md

@@ -1,6 +1,6 @@
 # TwitFix
 
-very basic flask server that fixes twitter embeds in discord by using youtube-dl to grab the direct link to the MP4 file and embeds the link to it in a custom page
+very basic flask server that fixes twitter embeds in discord by using youtube-dl to grab the direct link to the MP4 file and embeds the link to it in a custom page, with link caching
 
 This does work! but I'm new to flask, so it can probably be improved a great deal.
 
@@ -9,16 +9,23 @@ This does work! but I'm new to flask, so it can probably be improved a great dea
 just put the url to the server, and directly after, the full URL to the tweet you want to embed
 
 **I now have a copy of this running on a Linode server, you can use it via the following url**
-`http://twtfx.me/<twitter url>`
+
+https://fxtwitter.com/<twitter video url>
+
+You can also simply type out 'fx' directly before 'twitter.com' in any valid twitter video url, and that will convert it into a working TwitFix url, for example:
+
+![example](example.gif)
 
 **Note**: If you enjoy this service, please considering donating via [Ko-Fi](https://ko-fi.com/robin_universe) to help cover server costs
 
 ## How to run (server side)
 
-this script uses the youtube-dl python module, along with flask, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py` ( will need sudo if you leave it at port 80 )
+this script uses the youtube-dl python module, along with flask and pymongo, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py` ( will need sudo if you leave it at port 80 )
 
 By default I have the port set to 80, just cause that's what was convenient for me, but it can easily be changed, either using an environment variable, or changing the bottom line of the script itself
 
+I have included some files to give you a head start on setting this server up with uWSGI, though if you decide to use uWSGI I suggest you set up mongoDB link caching by going into the script and change `link_cache_system` from `json` to `"DB"`, and inserting you mongoDB address, as having many workers writing to the same json file doesn't really work
+
 This project is licensed under the **Do What The Fuck You Want Public License**
 
 

requirements.txt

@@ -1,2 +1,3 @@
 Flask
 youtube_dl
+pymongo

twitfix.ini

@@ -0,0 +1,11 @@
+[uwsgi]
+module = wsgi:app
+
+master = true
+processes = 5
+
+socket = 80
+chmod-socket = 660
+vacuum = true
+
+die-on-term = true

twitfix.py

@@ -1,4 +1,5 @@
 from flask import Flask, render_template
+import pymongo
 import youtube_dl
 import json
 import re
@@ -6,14 +7,20 @@ import re
 app = Flask(__name__)
 pathregex = re.compile("\\w{1,15}\\/status\\/\\d{19}")
 
-link_cache = {}
+link_cache_system = "json" # Select your prefered link cache system, "db" for mongoDB or "json" for locally writing a json file ( not great if using multiple workers )
-f = open('links.json',)
+
-link_cache = json.load(f)
+if link_cache_system == "json":
-f.close()
+    link_cache = {}
+    f = open('links.json',)
+    link_cache = json.load(f)
+    f.close()
+elif link_cache_system == "db":
+    client = pymongo.MongoClient("PUT YOUR MONGODB URL HERE")
+    db = client.TwitFix
 
 @app.route('/')
 def default():
-    return render_template('default.html', message="TwitFix is an attempt to fix twitter video embeds in discord click this link to be directed to my github page for the project!")
+    return render_template('default.html', message="TwitFix is an attempt to fix twitter video embeds in discord! created by Robin Universe :) 💖 ")
 
 @app.route('/<path:subpath>')
 def twitfix(subpath):
@@ -43,28 +50,65 @@ def info(subpath):
     return result
 
 def embedVideo(vidlink): # Return a render template from a video url
-    if vidlink in link_cache:
+    if link_cache_system == "db":
-        print("Link located in cache")
+        collection = db.linkCache
-        return render_template('index.html', vidurl=link_cache[vidlink]['url'], desc=link_cache[vidlink]['description'], pic=link_cache[vidlink]['thumbnail'], user=link_cache[vidlink]['uploader'], vidlink=vidlink)
+        dbresult = collection.find_one({'tweet': vidlink})
+        if dbresult != None:
+            print("Link located in DB cache")
+            return render_template('index.html', vidurl=dbresult['url'], desc=dbresult['description'], pic=dbresult['thumbnail'], user=dbresult['uploader'], vidlink=vidlink)
+        else:
+            with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
+                try:
+                    print("Link not in json cache, downloading and adding details to cache file")
+                    result = ydl.extract_info(vidlink, download=False)
+                    vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader'])
+
+                    try:
+                        out = db.linkCache.insert_one(vnf)
+                        print("Link added to DB cache")
+                    except Exception:
+                        print("Failed to add link to DB cache")
+                    
+                    return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
+
+                except Exception:
+                    print("Failed to download link")
+                    return render_template('default.html', message="Failed to scan your link!")
+    
+    elif link_cache_system == "json":
+        if vidlink in link_cache:
+            print("Link located in json cache")
+            return render_template('index.html', vidurl=link_cache[vidlink]['url'], desc=link_cache[vidlink]['description'], pic=link_cache[vidlink]['thumbnail'], user=link_cache[vidlink]['uploader'], vidlink=vidlink)
+        else:
+            with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
+                try:
+                    print("Link not in json cache, downloading and adding details to cache file")
+                    result = ydl.extract_info(vidlink, download=False)
+                    vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader'])
+                    link_cache[vidlink] = vnf
+
+                    with open("links.json", "w") as outfile: 
+                        json.dump(link_cache, outfile, indent=4, sort_keys=True)
+
+                except Exception: # Just to keep from 500s that are messy
+                    print("Failed to download link")
+                    return render_template('default.html', message="Failed to scan your link!")
+
+            return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
     else:
-        with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
+        try:
-            try:
+            with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
-                print("Link not in cache, downloading and adding details to cache file")
+                result = ydl.extract_info(subpath, download=False)
-                result = ydl.extract_info(vidlink, download=False)
+                vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader'])
-                vnf = vidInfo(result['url'], result['description'], result['thumbnail'], result['uploader'])
+                return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
-                link_cache[vidlink] = vnf
+        except Exception:
+            print("Failed to download link")
+            return render_template('default.html', message="Failed to scan your link!")
 
-                with open("links.json", "w") as outfile: 
-                    json.dump(link_cache, outfile, indent=4, sort_keys=True)
 
-            except Exception: # Just to keep from 500s that are messy
+def vidInfo(url, tweet="", desc="", thumb="", uploader=""): # Return a dict of video info with default values
-                print("Failed to download link")
-                return render_template('default.html', message="Failed to scan your link!")
-
-        return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
-
-def vidInfo(url, desc="", thumb="", uploader=""): # Return a dict of video info with default values
     vnf = {
+        "tweet"         :tweet,
         "url"           :url,
         "description"   :desc,
         "thumbnail"     :thumb,
@@ -73,4 +117,4 @@ def vidInfo(url, desc="", thumb="", uploader=""): # Return a dict of video info
     return vnf
 
 if __name__ == "__main__":
-    app.run(host='0.0.0.0', port=80)
+    app.run(host='0.0.0.0')

twitfix.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Init file for twitfix uwsgi instance
+After=network.target
+
+[Service]
+User=robin
+Group=robin
+WorkingDirectory=/home/robin/twitfix
+Environment="PATH=/home/robin/twitfix/twitfixenv/bin"
+ExecStart=/home/robin/twitfix/twitfixenv/bin/uwsgi --ini twitfix.ini
+
+[Install]
+WantedBy=multi-user.target

wsgi.py

@@ -0,0 +1,4 @@
+from twitfix import app
+
+if __name__ == "__main__":
+    app.run()