Add Docker image and compose with documentation

Minor cleaning

.gitignore

@@ -2,3 +2,10 @@ config.json
 __pycache__
 venv/
 links.json
+node_modules
+#SERVERLESS STUFF
+admin.env
+.env
+_meta
+.serverless
+db/

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.6-alpine AS build
+RUN apk add build-base python3-dev linux-headers pcre-dev jpeg-dev zlib-dev
+RUN pip install --upgrade pip
+RUN pip install yt-dlp pillow uwsgi
+
+FROM python:3.6-alpine AS deps
+WORKDIR /twitfix
+COPY requirements.txt requirements.txt
+COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
+RUN pip install -r requirements.txt
+
+FROM python:3.6-alpine AS runner
+EXPOSE 9000
+RUN apk add pcre-dev jpeg-dev zlib-dev
+WORKDIR /twitfix
+CMD ["uwsgi", "twitfix.ini"]
+COPY --from=build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
+COPY --from=deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
+COPY . .

combineImg/Dockerfile

@@ -0,0 +1,34 @@
+FROM public.ecr.aws/lambda/python:3.8
+RUN yum -y update
+RUN yum -y install git && yum clean all
+RUN yum -y install tar gzip zlib freetype-devel \
+    gcc \
+    ghostscript \
+    lcms2-devel \
+    libffi-devel \
+    libimagequant-devel \
+    libjpeg-devel \
+    libraqm-devel \
+    libtiff-devel \
+    libwebp-devel \
+    make \
+    openjpeg2-devel \
+    rh-python36 \
+    rh-python36-python-virtualenv \
+    sudo \
+    tcl-devel \
+    tk-devel \
+    tkinter \
+    which \
+    xorg-x11-server-Xvfb \
+    zlib-devel \
+    && yum clean all
+RUN pip install -U --force-reinstall pillow-simd
+RUN pip install requests
+
+
+# Copy function code
+COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "app.lambda_handler" ] 

combineImg/__init__.py

@@ -0,0 +1,140 @@
+from PIL import Image, ImageOps, ImageFilter
+from requests import get
+from io import BytesIO
+import base64
+import concurrent.futures
+from time import time as timer
+
+# find the highest res image in an array of images
+def findImageWithMostPixels(imageArray):
+    maxPixels = 0
+    maxImage = None
+    for image in imageArray:
+        pixels = image.size[0] * image.size[1]
+        if pixels > maxPixels:
+            maxPixels = pixels
+            maxImage = image
+    return maxImage
+
+def getTotalImgSize(imageArray): # take the image with the most pixels, multiply it by the number of images, and return the width and height
+    maxImage = findImageWithMostPixels(imageArray)
+    if (len(imageArray) == 1):
+        return (maxImage.size[0], maxImage.size[1])
+    elif (len(imageArray) == 2):
+        return (maxImage.size[0] * 2, maxImage.size[1])
+    else:
+        return (maxImage.size[0] * 2, maxImage.size[1]*2)
+
+def scaleImageIterable(args):
+    image = args[0]
+    targetWidth = args[1]
+    targetHeight = args[2]
+    pad=args[3]
+    if pad:
+        image = image.convert('RGBA')
+        newImg = ImageOps.pad(image, (targetWidth, targetHeight),color=(0, 0, 0, 0))
+    else:
+        newImg = ImageOps.fit(image, (targetWidth, targetHeight)) # scale + crop
+    return newImg
+
+def scaleAllImagesToSameSize(imageArray,targetWidth,targetHeight,pad=True): # scale all images in the array to the same size, preserving aspect ratio
+    newImageArray = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        newImageArray = [executor.submit(scaleImageIterable, (image, targetWidth, targetHeight,pad)) for image in imageArray]
+        newImageArray = [future.result() for future in newImageArray]
+    return newImageArray
+
+def blurImage(image, radius):
+    return image.filter(ImageFilter.GaussianBlur(radius=radius))
+
+def combineImages(imageArray, totalWidth, totalHeight,pad=True):
+    x = 0
+    y = 0
+    if (len(imageArray) == 1): # if there is only one image, just return it
+        return imageArray[0]
+    # image generation is needed
+    topImg = findImageWithMostPixels(imageArray)
+    newImage = Image.new("RGBA", (totalWidth, totalHeight),(0, 0, 0, 0))
+    imageArray = scaleAllImagesToSameSize(imageArray,topImg.size[0],topImg.size[1],pad)
+    if (len(imageArray) == 2): # if there are two images, combine them horizontally
+        for image in imageArray:
+            newImage.paste(image, (x, y))
+            x += image.size[0]
+    elif (len(imageArray) == 3): # the elusive 3 image upload
+        # if there are three images, combine the first two horizontally, then combine the last one vertically
+        imageArray[2] = scaleAllImagesToSameSize([imageArray[2]],totalWidth,topImg.size[1],pad)[0] # take the last image, treat it like an image array and scale it to the total width, but same height as all individual images
+        for image in imageArray[0:2]:
+            newImage.paste(image, (x, y))
+            x += image.size[0]
+        y += imageArray[0].size[1]
+        x = 0
+        newImage.paste(imageArray[2], (x, y))
+    elif (len(imageArray) == 4): # if there are four images, combine the first two horizontally, then combine the last two vertically
+        for image in imageArray[0:2]:
+            newImage.paste(image, (x, y))
+            x += image.size[0]
+        y += imageArray[0].size[1]
+        x = 0
+        for image in imageArray[2:4]:
+            newImage.paste(image, (x, y))
+            x += image.size[0]
+    else:
+        for image in imageArray:
+            newImage.paste(image, (x, y))
+            x += image.size[0]
+    return newImage
+
+def saveImage(image, name):
+    image.save(name)
+
+# combine up to four images into a single image
+def genImage(imageArray):
+    totalSize=getTotalImgSize(imageArray)
+    combined = combineImages(imageArray, *totalSize)
+    combinedBG = combineImages(imageArray, *totalSize,False)
+    combinedBG = blurImage(combinedBG,50)
+    finalImg = Image.alpha_composite(combinedBG,combined)
+    finalImg = ImageOps.pad(finalImg, findImageWithMostPixels(imageArray).size,color=(0, 0, 0, 0))
+    finalImg = finalImg.convert('RGB')
+    return finalImg
+
+def downloadImage(url):
+    return Image.open(BytesIO(get(url).content))
+
+def genImageFromURL(urlArray):
+    # this method avoids storing the images in disk, instead they're stored in memory
+    # no cache means that they'll have to be downloaded again if the image is requested again
+    # TODO: cache?
+    start = timer()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        imageArray = [executor.submit(downloadImage, url) for url in urlArray]
+        imageArray = [future.result() for future in imageArray]
+    print(f"Images downloaded in: {timer() - start}s")
+    start = timer()
+    finalImg = genImage(imageArray)
+    print(f"Image generated in: {timer() - start}s")
+    return finalImg
+    
+def lambda_handler(event, context):
+    if ("queryStringParameters" not in event):
+        return {
+            "statusCode": 400,
+            "body": "Invalid request."
+        }
+    images = event["queryStringParameters"].get("imgs","").split(",")
+    for img in images:
+        if not img.startswith("https://pbs.twimg.com"):
+            return {'statusCode':400,'body':'Invalid image URL'}
+    combined = genImageFromURL(images)
+    buffered = BytesIO()
+    combined.save(buffered,format="JPEG",quality=60)
+    combined_str=base64.b64encode(buffered.getvalue()).decode('ascii')
+    return {
+        'statusCode': 200,
+        "headers": 
+        {
+            "Content-Type": "image/jpeg"
+        },
+        'body': combined_str,
+        'isBase64Encoded': True
+    }

docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3"
+
+services:
+  proxy:
+    image: nginx:alpine
+    container_name: twitfix_proxy
+    volumes:
+      - "./twitfix_proxy.conf:/etc/nginx/conf.d/default.conf"
+    ports:
+      - 8088:80
+    depends_on:
+      - twitfix
+
+  twitfix:
+    image: twitfix
+    build: .
+    container_name: twitfix_main
+    volumes:
+      - "./twitfix.ini:/twitfix/twitfix.ini:ro"
+      - "./config.json:/twitfix/config.json:ro"
+    depends_on:
+      - db
+
+  db:
+    image: mongo:5.0.9
+    container_name: twitfix_db
+    volumes:
+      - "./db:/data/db"
+

docker.md

@@ -0,0 +1,40 @@
+# vxTwitter Docker
+
+## Configuration
+
+Setup mongodb in `config.json`:
+```json
+{
+    "config":{
+        "link_cache":"db",
+        "database":"mongodb://twitfix_db:27017/",
+        [...]
+    },
+    [...]
+}
+```
+
+Use TCP socket for uwsgi `twitfix.ini`:
+```ini
+[uwsgi]
+module = wsgi:app
+
+master = true
+processes = 5
+
+socket = 0.0.0.0:9000
+buffer-size = 8192
+#socket = /var/run/twitfix.sock
+#chmod-socket = 660
+vacuum = true
+
+die-on-term = true
+```
+
+## Run
+
+To run and build, use this command:
+```bash
+docker-compose up -d --build
+```
+

msgs.py

@@ -0,0 +1 @@
+failedToScan="Failed to scan your link! This may be due to an incorrect link, private account, or the twitter API itself might be having issues (Check here: https://api.twitterstat.us/)\nIt's also possible that Twitter is API limiting me, in which case I can't do anything about it."

readme.md

@@ -2,6 +2,11 @@
 (A fork of TwitFix)
 Basic flask server that serves fixed twitter video embeds to desktop discord by using either the Twitter API or Youtube-DL to grab tweet video information. This also automatically embeds the first link in the text of non video tweets (API Only)
 
+## Differences from fxtwitter
+fxtwitter exposed all recently processed tweets publicly via a "latest" and "top" page.
+
+Even though Tweets are public, it was a personal concern for me that a tweet with potentially sensitive information in it could suddenly be shown to however many people were browsing the latest tweets page, and could be used as a tool for harassment. This was removed in [The following commit](https://github.com/dylanpdx/BetterTwitFix/commit/87ba86ba502e73ddb370bd4e5b964548d3272400#diff-a11c36d9b2d53672d6b3d781dca5bef9129159947de66bc3ffaec5fab389d80cL115)
+
 ## How to use (discord side)
 
 just put the url to the server, and directly after, the full URL to the tweet you want to embed
@@ -16,6 +21,8 @@ You can also simply type out 'vx' directly before 'twitter.com' in any valid twi
 
 **Note**: If you enjoy this service, please considering donating via [Ko-Fi](https://ko-fi.com/dylanpdx) to help cover server costs
 
+I do not monitor any tweets processed by this server. Additionally, if you plan on hosting the code yourself and are concerned about this, be sure to check how to disable logging on the web server you are using (i.e Nginx)
+
 ## How to run (server side)
 
 this script uses the youtube-dl python module, along with flask, twitter and pymongo, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py`
@@ -50,6 +57,8 @@ vxTwitter generates a config.json in its root directory the first time you run i
 
 **url** - used to tell the user where to look for the oembed endpoint, make sure to set this to your public facing url
 
+**combination_method** - using c.vxtwitter as the url causes vxTwitter to combine all images in the post into one. This is CPU intensive, so you might not want it running on the same machine that's serving requests. When `combination_method` is set to `local`, it will use the local machine to combine the images. This requires pillow to be installed. If you want to use another server, replace `local` with the URL to the endpoint which combines images. Both methods use the code in the `combineImg` module. Inside, there's also a `Dockerfile` intended to be deployed as a combination endpoint on an [AWS Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html).
+
 This project is licensed under the **Do What The Fuck You Want Public License**
 
 

templates/image.html

@@ -14,22 +14,17 @@
     {{ pfp }}     - URL of the posters PFP
  -->
 {% block head %}
-    <meta content="text/html; charset=UTF-8"        http-equiv="Content-Type" />
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
-    <meta content="{{ color }}"                     name="theme-color" />
+<meta content="{{ color }}" name="theme-color" />
-    <meta property="og:site_name"                   content="{{ appname }}"/>
+<meta property="og:site_name" content="{{ appname }}" />
 
-    <meta name="twitter:card"                       content="summary_large_image" />
+<meta name="twitter:card" content="summary_large_image" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta name="twitter:creator"                    content="@{{ user }}" />
+<meta name="twitter:creator" content="@{{ user }}" />
 
-    <meta property="og:description"                 content="{{ desc }}"  />
+<meta property="og:description" content="{{ desc }}" />
 
-    <!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
+<!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

templates/index.html

@@ -1,37 +1,30 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}">
 
-{% block head %}
+<meta name="twitter:card" content="player" />
-    <meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta name="twitter:title" content="{{ user }}" />
-    <meta content="{{ color }}" name="theme-color" />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta property="og:site_name" content="{{ appname }}">
+<!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
-
-    <meta name="twitter:card"                       content="player" />
-    <meta name="twitter:title"                      content="{{ user }}" />
-    <meta name="twitter:image"                      content="{{ pic }}" />
-    <!-- The line below breaks Telegram embeds if present but doesn't affect Discord embeds if removed
     <meta name="twitter:player"                     content="{{ vidurl }}" />
     <meta name="twitter:description"                content="{{ desc }}" />!-->
-    <meta name="twitter:player:width"               content="720" />
+<meta name="twitter:player:width" content="720" />
-    <meta name="twitter:player:height"              content="480" />
+<meta name="twitter:player:height" content="480" />
-    <meta name="twitter:player:stream"              content="{{ vidurl }}" />
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
-    <meta name="twitter:player:stream:content_type" content="video/mp4" /> 
+<meta name="twitter:player:stream:content_type" content="video/mp4" />
 
 
-    <meta property="og:url"                content="{{ vidlink }}" />
+<meta property="og:url" content="{{ vidlink }}" />
-    <meta property="og:video"              content="{{ vidurl }}" />
+<meta property="og:video" content="{{ vidurl }}" />
-    <meta property="og:video:secure_url"   content="{{ vidurl }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
-    <meta property="og:video:type"         content="video/mp4" />
+<meta property="og:video:type" content="video/mp4" />
-    <meta property="og:video:width"        content="720" />
+<meta property="og:video:width" content="720" />
-    <meta property="og:video:height"       content="480" />
+<meta property="og:video:height" content="480" />
-    <meta property="og:title"              content="{{ user }}" />
+<meta property="og:title" content="{{ user }}" />
-    <!--<meta property="og:description"        content="{{ desc }}" />!-->
+<!--<meta property="og:description"        content="{{ desc }}" />!-->
-    <meta property="og:image"              content="{{ pic }}" />
+<meta property="og:image" content="{{ pic }}" />
 
-        <!--!-->
+<!--!-->
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

templates/text.html

@@ -18,22 +18,17 @@
     {{ time }}    - Time Created
  -->
 {% block head %}
-    <meta content="text/html; charset=UTF-8"        http-equiv="Content-Type" />
+<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
-    <meta content="{{ color }}"                     name="theme-color" />
+<meta content="{{ color }}" name="theme-color" />
-    <meta property="og:site_name"                   content="{{ appname }}"/>
+<meta property="og:site_name" content="{{ appname }}" />
-    <meta property="og:image"                       content="{{ pfp }}"   />
+<meta property="og:image" content="{{ pfp }}" />
 
-    <meta name="twitter:card"                       content="tweet"       />
+<meta name="twitter:card" content="tweet" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
-    <meta name="twitter:image"                      content="{{ pic }}"   />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta name="twitter:creator"                    content="@{{ user }}" />
+<meta name="twitter:creator" content="@{{ user }}" />
 
-    <meta property="og:description"                 content="{{ desc }}"  />
+<meta property="og:description" content="{{ desc }}" />
 
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

templates/video.html

@@ -1,31 +1,25 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta content="{{ color }}" name="theme-color" />
+<meta property="og:site_name" content="{{ appname }}">
 
-{% block head %}
+<meta name="twitter:card" content="player" />
-    <meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
-    <meta content="{{ color }}"  name="theme-color" />
+<meta name="twitter:image" content="{{ pic }}" />
-    <meta property="og:site_name" content="{{ appname }}">
+<meta name="twitter:player:width" content="720" />
+<meta name="twitter:player:height" content="480" />
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
+<meta name="twitter:player:stream:content_type" content="video/mp4" />
 
-    <meta name="twitter:card"                       content="player" />
+<meta property="og:url" content="{{ vidlink }}" />
-    <meta name="twitter:title"                      content="{{ user }} (@{{ screenName }})"  />
+<meta property="og:video" content="{{ vidurl }}" />
-    <meta name="twitter:image"                      content="{{ pic }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
-    <meta name="twitter:player:width"               content="720" />
+<meta property="og:video:type" content="video/mp4" />
-    <meta name="twitter:player:height"              content="480" />
+<meta property="og:video:width" content="720" />
-    <meta name="twitter:player:stream"              content="{{ vidurl }}" />
+<meta property="og:video:height" content="480" />
-    <meta name="twitter:player:stream:content_type" content="video/mp4" /> 
+<meta name="twitter:title" content="{{ user }} (@{{ screenName }})" />
+<meta property="og:image" content="{{ pic }}" />
+<meta property="og:description" content="{{ desc }}" />
 
-    <meta property="og:url"                content="{{ vidlink }}" />
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
-    <meta property="og:video"              content="{{ vidurl }}" />
+<meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}
-    <meta property="og:video:secure_url"   content="{{ vidurl }}" />
-    <meta property="og:video:type"         content="video/mp4" />
-    <meta property="og:video:width"        content="720" />
-    <meta property="og:video:height"       content="480" />
-    <meta name="twitter:title"             content="{{ user }} (@{{ screenName }})"  />
-    <meta property="og:image"              content="{{ pic }}" />
-
-    <link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
-    <meta http-equiv = "refresh" content = "0; url = {{ vidlink }}" />
-{% endblock %}
-
-{% block body %}
-Redirecting you to the tweet in a moment. <a href="{{ vidlink }}">Or click here.</a>
-{% endblock %}

twExtract.py

@@ -0,0 +1,18 @@
+import imp
+import yt_dlp
+from yt_dlp.extractor import twitter
+import json
+
+def extractStatus(url):
+    twIE = twitter.TwitterIE()
+    twIE.set_downloader(yt_dlp.YoutubeDL())
+    twid = twIE._match_id(url)
+    status = twIE._call_api(
+    'statuses/show/%s.json' % twid, twid, {
+        'cards_platform': 'Web-12',
+        'include_cards': 1,
+        'include_reply_count': 1,
+        'include_user_entities': 0,
+        'tweet_mode': 'extended',
+    })
+    return status

twitfix.py

@@ -1,6 +1,7 @@
-from flask import Flask, render_template, request, redirect, Response, send_from_directory, url_for, send_file, make_response, jsonify
+from weakref import finalize
+from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, url_for, send_file, make_response, jsonify
 from flask_cors import CORS
-import youtube_dl
+import yt_dlp
 import textwrap
 import twitter
 import pymongo
@@ -10,7 +11,11 @@ import re
 import os
 import urllib.parse
 import urllib.request
-from datetime import date
+import combineImg
+from datetime import date,datetime, timedelta
+from io import BytesIO
+import msgs
+import twExtract
 
 app = Flask(__name__)
 CORS(app)
@@ -43,7 +48,8 @@ if not os.path.exists("config.json"):
                 "color":"#43B581", 
                 "appname": "vxTwitter", 
                 "repo": "https://github.com/dylanpdx/BetterTwitFix", 
-                "url": "https://vxtwitter.com"
+                "url": "https://vxtwitter.com",
+                "combination_method": "local" # can either be 'local' or a URL to a server handling requests in the same format
                 },
             "api":{"api_key":"[api_key goes here]",
             "api_secret":"[api_secret goes here]",
@@ -74,9 +80,15 @@ if link_cache_system == "json":
             default_link_cache = {"test":"test"}
             json.dump(default_link_cache, outfile, indent=4, sort_keys=True)
 
-    f = open('links.json',)
+    try:
-    link_cache = json.load(f)
+        with open('links.json', "r") as f:
-    f.close()
+            link_cache = json.load(f)
+    except (json.decoder.JSONDecodeError, FileNotFoundError):
+        print(" ➤ [ X ] Failed to load cache JSON file. Creating new file.")
+        with open('links.json', "w") as f:
+            link_cache = {}
+            json.dump(link_cache, f)
+
 elif link_cache_system == "db":
     client = pymongo.MongoClient(config['config']['database'], connect=False)
     table = config['config']['table']
@@ -105,16 +117,28 @@ def twitfix(sub_path):
     print(request.url)
 
     if request.url.startswith("https://d.vx"): # Matches d.fx? Try to give the user a direct link
+        if match.start() == 0:
+            twitter_url = "https://twitter.com/" + sub_path
         if user_agent in generate_embed_user_agents:
             print( " ➤ [ D ] d.vx link shown to discord user-agent!")
             if request.url.endswith(".mp4") and "?" not in request.url:
-                return dl(sub_path)
+                return redirect(direct_video_link(twitter_url),302)
             else:
                 return message("To use a direct MP4 link in discord, remove anything past '?' and put '.mp4' at the end")
         else:
             print(" ➤ [ R ] Redirect to MP4 using d.fxtwitter.com")
             return dir(sub_path)
+    elif request.url.startswith("https://c.vx"):
+        twitter_url = sub_path
 
+        if match.start() == 0:
+            twitter_url = "https://twitter.com/" + sub_path
+        
+        if user_agent in generate_embed_user_agents:
+            return embedCombined(twitter_url)
+        else:
+            print(" ➤ [ R ] Redirect to " + twitter_url)
+            return redirect(twitter_url, 301)
     elif request.url.endswith(".mp4") or request.url.endswith("%2Emp4"):
         twitter_url = "https://twitter.com/" + sub_path
         
@@ -123,7 +147,7 @@ def twitfix(sub_path):
         else:
             clean = twitter_url
 
-        return dl(clean)
+        return redirect(direct_video_link(clean),302)
 
     # elif request.url.endswith(".json") or request.url.endswith("%2Ejson"):
     #     twitter_url = "https://twitter.com/" + sub_path
@@ -196,6 +220,33 @@ def favicon():
     return send_from_directory(os.path.join(app.root_path, 'static'),
                           'favicon.ico',mimetype='image/vnd.microsoft.icon')
 
+@app.route("/rendercombined.jpg")
+def rendercombined():
+    # get "imgs" from request arguments
+    imgs = request.args.get("imgs", "")
+
+    if 'combination_method' in config['config'] and config['config']['combination_method'] != "local":
+        url = config['config']['combination_method'] + "/rendercombined.jpg?imgs=" + imgs
+        return redirect(url, 302)
+    # Redirecting here instead of setting the embed URL directly to this because if the config combination_method changes in the future, old URLs will still work
+
+    imgs = imgs.split(",")
+    if (len(imgs) == 0 or len(imgs)>4):
+        abort(400)
+    #check that each image starts with "https://pbs.twimg.com"
+    for img in imgs:
+        if not img.startswith("https://pbs.twimg.com"):
+            abort(400)
+    finalImg= combineImg.genImageFromURL(imgs)
+    imgIo = BytesIO()
+    finalImg = finalImg.convert("RGB")
+    finalImg.save(imgIo, 'JPEG',quality=70)
+    imgIo.seek(0)
+    return send_file(imgIo, mimetype='image/jpeg')
+
+def getDefaultTTL():
+    return datetime.today().replace(microsecond=0) + timedelta(days=1)
+
 def direct_video(video_link): # Just get a redirect to a MP4 link from any tweet link
     cached_vnf = getVnfFromLinkCache(video_link)
     if cached_vnf == None:
@@ -206,7 +257,7 @@ def direct_video(video_link): # Just get a redirect to a MP4 link from any tweet
             print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
         except Exception as e:
             print(e)
-            return message("Failed to scan your link!")
+            return message(msgs.failedToScan)
     else:
         return redirect(cached_vnf['url'], 301)
         print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
@@ -221,7 +272,7 @@ def direct_video_link(video_link): # Just get a redirect to a MP4 link from any
             print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
         except Exception as e:
             print(e)
-            return message("Failed to scan your link!")
+            return message(msgs.failedToScan)
     else:
         return cached_vnf['url']
         print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
@@ -237,11 +288,13 @@ def embed_video(video_link, image=0): # Return Embed from any tweet link
 
         except Exception as e:
             print(e)
-            return message("Failed to scan your link!")
+            return message(msgs.failedToScan)
     else:
         return embed(video_link, cached_vnf, image)
 
-def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False): # Return a dict of video info with default values
+def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None): # Return a dict of video info with default values
+    if (ttl==None):
+        ttl = getDefaultTTL()
     vnf = {
         "tweet"         : tweet,
         "url"           : url,
@@ -257,16 +310,19 @@ def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp
         "rts"           : rts,
         "time"          : time,
         "qrt"           : qrt,
-        "nsfw"          : nsfw
+        "nsfw"          : nsfw,
+        "ttl"           : ttl
     }
     return vnf
 
-def link_to_vnf_from_api(video_link):
+def get_tweet_data_from_api(video_link):
     print(" ➤ [ + ] Attempting to download tweet info from Twitter API")
     twid = int(re.sub(r'\?.*$','',video_link.rsplit("/", 1)[-1])) # gets the tweet ID as a int from the passed url
     tweet = twitter_api.statuses.show(_id=twid, tweet_mode="extended")
-    # For when I need to poke around and see what a tweet looks like
+    #print(tweet) # For when I need to poke around and see what a tweet looks like
-    #print(tweet)
+    return tweet
+
+def link_to_vnf_from_tweet_data(tweet,video_link):
     imgs = ["","","","", ""]
     print(" ➤ [ + ] Tweet Type: " + tweetType(tweet))
     # Check to see if tweet has a video, if not, make the url passed to the VNF the first t.co link in the tweet
@@ -277,6 +333,7 @@ def link_to_vnf_from_api(video_link):
             for video in tweet['extended_entities']['media'][0]['video_info']['variants']:
                 if video['content_type'] == "video/mp4" and video['bitrate'] > best_bitrate:
                     url = video['url']
+                    best_bitrate = video['bitrate']
     elif tweetType(tweet) == "Text":
         url   = ""
         thumb = ""
@@ -325,9 +382,21 @@ def link_to_vnf_from_api(video_link):
         
     return vnf
 
+
+def link_to_vnf_from_unofficial_api(video_link):
+    print(" ➤ [ + ] Attempting to download tweet info from UNOFFICIAL Twitter API")
+    tweet = twExtract.extractStatus(video_link)
+    print (" ➤ [ ✔ ] Unofficial API Success")
+    return link_to_vnf_from_tweet_data(tweet,video_link)
+
+
+def link_to_vnf_from_api(video_link):
+    tweet = get_tweet_data_from_api(video_link)
+    return link_to_vnf_from_tweet_data(tweet,video_link)
+
 def link_to_vnf_from_youtubedl(video_link):
     print(" ➤ [ X ] Attempting to download tweet info via YoutubeDL: " + video_link)
-    with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
+    with yt_dlp.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
         result = ydl.extract_info(video_link, download=False)
         vnf    = tweetInfo(result['url'], video_link, result['description'].rsplit(' ',1)[0], result['thumbnail'], result['uploader'])
         return vnf
@@ -339,7 +408,13 @@ def link_to_vnf(video_link): # Return a VideoInfo object or die trying
         except Exception as e:
             print(" ➤ [ !!! ] API Failed")
             print(e)
-            return link_to_vnf_from_youtubedl(video_link)
+            try:
+                return link_to_vnf_from_unofficial_api(video_link)
+            except Exception as e:
+                print(" ➤ [ !!! ] UNOFFICIAL API Failed")
+                print(e)
+                return link_to_vnf_from_youtubedl(video_link) # This is the last resort, will only work for videos
+                
     elif config['config']['method'] == 'api':
         try:
             return link_to_vnf_from_api(video_link)
@@ -382,20 +457,25 @@ def getVnfFromLinkCache(video_link):
             print(" ➤ [ X ] Link not in json cache")
             return None
 
+def serializeUnknown(obj):
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    raise TypeError ("Type %s not serializable" % type(obj))
+
 def addVnfToLinkCache(video_link, vnf):
-    if link_cache_system == "db":
+    try:
-        try:
+        if link_cache_system == "db":
-            out = db.linkCache.insert_one(vnf)
+                out = db.linkCache.insert_one(vnf)
-            print(" ➤ [ + ] Link added to DB cache ")
+                print(" ➤ [ + ] Link added to DB cache ")
-            return True
+                return True
-        except Exception:
+        elif link_cache_system == "json":
-            print(" ➤ [ X ] Failed to add link to DB cache")
+            link_cache[video_link] = vnf
-            return None
+            with open("links.json", "w") as outfile: 
-    elif link_cache_system == "json":
+                json.dump(link_cache, outfile, indent=4, sort_keys=True, default=serializeUnknown)
-        link_cache[video_link] = vnf
+                return None
-        with open("links.json", "w") as outfile: 
+    except Exception:
-            json.dump(link_cache, outfile, indent=4, sort_keys=True)
+        print(" ➤ [ X ] Failed to add link to DB cache")
-            return None
+        return None
 
 def message(text):
     return render_template(
@@ -428,10 +508,12 @@ def embed(video_link, vnf, image):
     except:
         vnf['likes'] = 0; vnf['rts'] = 0; vnf['time'] = 0
         print(' ➤ [ X ] Failed QRT check - old VNF object')
-        
+    appNamePost = ""
     if vnf['type'] == "Text": # Change the template based on tweet type
         template = 'text.html'
     if vnf['type'] == "Image":
+        if vnf['images'][4]!="1":
+            appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4])
         image = vnf['images'][image]
         template = 'image.html'
     if vnf['type'] == "Video":
@@ -460,12 +542,75 @@ def embed(video_link, vnf, image):
         user       = vnf['uploader'], 
         video_link = video_link, 
         color      = color, 
-        appname    = config['config']['appname'], 
+        appname    = config['config']['appname']+appNamePost, 
         repo       = config['config']['repo'], 
         url        = config['config']['url'], 
         urlDesc    = urlDesc, 
         urlUser    = urlUser, 
-        urlLink    = urlLink )
+        urlLink    = urlLink,
+        tweetLink  = vnf['tweet'] )
+
+
+def embedCombined(video_link):
+    cached_vnf = getVnfFromLinkCache(video_link)
+
+    if cached_vnf == None:
+        try:
+            vnf = link_to_vnf(video_link)
+            addVnfToLinkCache(video_link, vnf)
+            return embedCombinedVnf(video_link, vnf)
+
+        except Exception as e:
+            print(e)
+            return message(msgs.failedToScan)
+    else:
+        return embedCombinedVnf(video_link, cached_vnf)
+
+def embedCombinedVnf(video_link,vnf):
+    if vnf['type'] != "Image" or vnf['images'][4] == "1":
+        return embed(video_link, vnf, 0)
+    desc    = re.sub(r' http.*t\.co\S+', '', vnf['description'])
+    urlUser = urllib.parse.quote(vnf['uploader'])
+    urlDesc = urllib.parse.quote(desc)
+    urlLink = urllib.parse.quote(video_link)
+    likeDisplay = ("\n\n💖 " + str(vnf['likes']) + " 🔁 " + str(vnf['rts']) + "\n")
+
+    if vnf['qrt'] == {}: # Check if this is a QRT and modify the description
+            desc = (desc + likeDisplay)
+    else:
+        qrtDisplay = ("\n─────────────\n ➤ QRT of " + vnf['qrt']['handle'] + " (@" + vnf['qrt']['screen_name'] + "):\n─────────────\n'" + vnf['qrt']['desc'] + "'")
+        desc = (desc + qrtDisplay +  likeDisplay)
+
+    color = "#7FFFD4" # Green
+
+    if vnf['nsfw'] == True:
+        color = "#800020" # Red
+    image = "https://vxtwitter.com/rendercombined.jpg?imgs="
+    for i in range(0,int(vnf['images'][4])):
+        image = image + vnf['images'][i] + ","
+    image = image[:-1] # Remove last comma
+    return render_template(
+        'image.html', 
+        likes      = vnf['likes'], 
+        rts        = vnf['rts'], 
+        time       = vnf['time'], 
+        screenName = vnf['screen_name'], 
+        vidlink    = vnf['url'], 
+        pfp        = vnf['pfp'],  
+        vidurl     = vnf['url'], 
+        desc       = desc,
+        pic        = image,
+        user       = vnf['uploader'], 
+        video_link = video_link, 
+        color      = color, 
+        appname    = config['config']['appname'] + " - View original tweet for full quality", 
+        repo       = config['config']['repo'], 
+        url        = config['config']['url'], 
+        urlDesc    = urlDesc, 
+        urlUser    = urlUser, 
+        urlLink    = urlLink,
+        tweetLink  = vnf['tweet'] )
+
 
 def tweetType(tweet): # Are we dealing with a Video, Image, or Text tweet?
     if 'extended_entities' in tweet:

twitfix.service

@@ -8,6 +8,8 @@ Group=dylan
 WorkingDirectory=/home/dylan/BetterTwitFix
 Environment="PATH=/home/dylan/BetterTwitFix/venv/bin"
 ExecStart=/home/dylan/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
+Restart=always
+RestartSec=3
 
 [Install]
 WantedBy=multi-user.target

twitfix_proxy.conf

@@ -0,0 +1,24 @@
+server {
+    listen       80;
+    server_name  localhost;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        try_files $uri @twitfix;
+    }
+
+    location @twitfix {
+        include uwsgi_params;
+        uwsgi_pass uwsgi://twitfix_main:9000;
+    }
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}