# -*- coding: utf-8 -*-

# artetv.py
# This file is part of qarte-5
#    
# Author: Vincent Vande Vyvre <vincent.vandevyvre@oqapy.eu>
# Copyright: 2011-2025 Vincent Vande Vyvre
# Licence: GPL3
# Home page: https://launchpad.net/qarte
#
# arte TV Guide main class


import os
import sys
import shutil
import time
import json
import glob
import subprocess
import urllib.request
import urllib.parse
import html
import re
import logging
lgg = logging.getLogger(__name__)

from datetime import datetime, date, timedelta
from threading import Thread
from html.parser import HTMLParser
 
from PyQt5.QtGui import QFont, QTextCharFormat
from PyQt5.QtCore import QObject, QCoreApplication, pyqtSignal, QLocale, QSize
import m3u8

from data import ARTE_TV, USER_AGENT
from gui.tvdownloadingconfig import TvDownloadingConfig
from downloader import Downloader
from gui.warning import WarningBox


class ArteTV(QObject):
    """Define the arte TV Guide server handler.

    """
    parsingFinished = pyqtSignal(int)
    mergingFinished = pyqtSignal(str, str)
    def __init__(self, core):
        super().__init__()
        self.core = core
        self.ui = core.ui
        self.lang = self.core.cfg.get('lang')
        self.fatal_error = False
        self.cache = core.workspace['tv_tmp']
        self.temp_dir = core.workspace['temp']
        self.videos = []
        self.is_loading = False
        self.is_in_category = False
        self.current_category = "arte+7"
        lang = QLocale.system().name()
        self.locale = QLocale(lang)
        self.downloader = Downloader(self)
        self.parsingFinished.connect(self.load_extras)
        self.ui.tv_basket.itemAdded.connect(self.on_item_added)

    def config_parser(self):
        """Configure the main pages parser.

        """
        self.run_html_parser()

    def run_html_parser(self):
        begin = time.time()
        hparser = HtmlParser(ARTE_TV.replace('fr', self.lang), self.lang)
        self.videos = []
        self.indexes = []
        for i in range(10):
            hparser.videos = []
            content = hparser.get_next_page()
            if content:
                hparser.feed(content)
                for v in hparser.videos:
                    idx = self.extract_index(v["url"])
                    self.indexes.insert(0, idx)
                    v["id"] = idx
                    self.videos.insert(0, TVItem(v))

            else:
                log.warning("Can't get the page !")

        self.check_thumbnails()
        videos_count = len(self.videos)
        end = time.time() - begin
        lgg.info('Found %s videos in %s sec.' % (videos_count, end))
        self.parsingFinished.emit(videos_count)

    def extract_index(self, url):
        return url.split("/")[3]

    def sort_videos(self, news, idxs):
        """Create the sorted (youngest to oldest) list of the videos.

        """
        lgg.info('Sorting videos')
        while news:
            item = news.pop(0)
            if item["programId"] not in self.indexes:
                self.videos.insert(0, TVItem(item))
                self.indexes.insert(0, item["programId"])

    def load_extras(self, number):
        """Load the list of videos by category.

        Args:
        number -- number of videos in arte+7, unusued
        """
        url = 'http://www.oqapy.eu/htdocs/releases/artetv_videos_%s.jsn' % self.lang
        update = self.get_update(url)
        is_videos = False
        try:
            self.categories_videos = json.loads(update)
        except Exception as why:
            lgg.warning("Unable to read the arte tv list")
            lgg.warning("Reason: %s" % why)
            fl = self.core.workspace['user'] + '/artetv_videos_%s.jsn' % self.lang
            try:
                with open(fl, "rb") as inf:
                    update = inf.read()
                self.categories_videos =  json.loads(update)
            except Exception as why:
                lgg.warning("Unable to read the arte tv list")
                lgg.warning("Reason: %s" % why)
                return

        lgg.info('arte tv list updated')
        self.categories = [(k, "") for k in self.categories_videos.keys()]
        self.categories.insert(0, ["arte+7", ""])
        self.clean_lists()
        self.ui.set_categories_items(self.categories)
        is_videos = True
        self.ui.categories_cmb.setEnabled(is_videos)
        if is_videos:
            self.ui.categories_cmb.currentTextChanged.connect(
                                        self.change_category)
            self.ui.refresh_btn.clicked.connect(self.reload_category)

        self.remove_old_videos()

    def get_update(self, url):
        req = urllib.request.Request(url, data=None,
                                     headers={"User-Agent": USER_AGENT})
        lgg.info('Get update ...')
        try:
            content = urllib.request.urlopen(req, timeout=5)
            return str(content.read().decode('utf-8', 'replace'))
        except Exception as why:
            lgg.info('urllib in update artetv error: %s, %s' % (url, why))
            return False

    def clean_lists(self):
        discarded = ("Les vidéos les plus vues", "Parcourir les catégories",
                     "Meistgesehene Videos", "Alle Kategorien")

        for d in discarded:
            for idx, c in enumerate(self.categories):
                if d in c:
                    self.categories.pop(idx)
                    self.categories_videos.pop(d)
                    break

    def remove_old_videos(self):
        def remove():
            self.clean_thumbnails()

        Thread(target=remove).start()

    def clean_thumbnails(self):
        """Remove old thumbnails from the folder 'plusPreviews'.

        """
        lgg.info("Clean thumbnail folder")
        fld = self.core.workspace['tv_tmp']
        thumbs = glob.glob(os.path.join(fld, '*.*'))
        for thumb in thumbs:
            index = os.path.basename(thumb).split('.')[0]
            if not index in self.indexes:
                try:
                    os.remove(thumb)
                except Exception as why:
                    lgg.warning("Can't remove thumbnail: %s" % thumb)
                    lgg.warning("Reason: %s" % why)

    def check_thumbnails(self):
        """Verify if all thumbnails exists.

        """
        lgg.info('Check thumbnails')
        count = 0
        for idx, index, in enumerate(self.indexes):
            image = os.path.join(self.cache, "%s.jpg" % index)
            image2 = os.path.join(self.cache, "%s.JPG" % index)
            if not (os.path.isfile(image) or os.path.isfile(image2)):
                image = self.videos[idx].get_pixmap()
                if image is None:
                    image = self.load_image(self.videos[idx])
                    count += 1
            
            self.videos[idx].pixmap = image
        lgg.info('Thumbnails completed, %s news items.' % count)

    def edit_summary(self, item):
        """Show the summary.

        Args:
        idx -- video item 
        """
        if not item.label:
            self.get_item_data(item)
        self.ui.pitch_editor.clear()
        font = QFont("sans", 10)
        c_form = QTextCharFormat()
        c_form.setFont(font)
        font1 = QFont("sans", 12)
        font1.setWeight(75)
        c_form1 = QTextCharFormat()
        c_form1.setFont(font1)
        self.ui.pitch_editor.setCurrentCharFormat(c_form1)
        self.ui.pitch_editor.appendPlainText(item.title)
        self.ui.pitch_editor.setCurrentCharFormat(c_form)
        t = "  %s\n %s %s\n" % (item.duration, item.origin, item.anno)
        self.ui.pitch_editor.insertPlainText(t)
        if item.realisation:
            rea = ", ".join(item.realisation)
            self.ui.pitch_editor.appendPlainText("  Real: %s" % rea)
        if item.music:
            music = ", ".join(item.music)
            self.ui.pitch_editor.appendPlainText("  Music: %s" % music)
        desc = item.description
        if not desc:
            desc = item.teaser
            if not desc:
                desc = "No description."
        self.ui.pitch_editor.appendPlainText('')
        if "</" in desc:
            self.ui.pitch_editor.appendHtml("    %s" % desc)

        else:
            self.ui.pitch_editor.appendPlainText("    %s" % desc)

        if item.actors:
            self.ui.pitch_editor.appendPlainText('')
            self.ui.pitch_editor.appendPlainText(" Cast: ")
            actors = "\n    ".join(item.actors)
            self.ui.pitch_editor.appendPlainText("    %s" % actors)

        self.ui.pitch_editor.verticalScrollBar().setValue(0)
        QCoreApplication.processEvents()
        item.summary = self.ui.pitch_editor.toPlainText()

    def get_item_data(self, item):
        """Fetch the long description of a video.

        If the long description can't be found the short one is used or
        at least the teaser which always exists.

        Args:
        item -- TVItem instance
        """
        lgg.info("Download item page: %s" % item.url)
        if item.url.startswith(("/fr/", "/de/")):
            item.url = "https://www.arte.tv" + item.url
        item.player = "https://api.arte.tv/api/player/v2/config/%s/%s" \
                        %(self.lang, item.idx)
        page = self.get_page_content(item.url)
        if page:
            if self.current_category == "Séries":
                if not "RC-" in item.url:
                    self.get_serial_item(item, page)
                    return

                else:
                    self.get_serial_collection(item, page)
                    return

            parser = ItemParser()
            parser.feed(page)
            data = parser.item
            if data["desc"]:
                item.description = data["desc"]
            try:
                if data["teaser"]:
                    item.teaser = data["teaser"]
            except KeyError:
                pass

            item.duration = data["duration"]
            item.origin = ", ".join(data["country"])
            item.anno = data["date"]
            item.realisation = data["real"]
            item.actors = data["cast"]
            self.get_item_description(item)

    def get_serial_item(self, item, content):
        lgg.info("Get serial item")
        try:
            data = json.loads(content)
        except Exception as why:
            lgg.info("json read error: %s" % why)
            return

        meta = data["data"]["attributes"]["metadata"]
        item.idx = meta["providerId"]
        item.title = meta["title"]
        item.description = meta["description"]
        item.duration = item.format_duration(meta["duration"]["seconds"])
        streams, error = self.read_streams(content)
        if streams:
            labels = self.sort_streams_by_label(streams)
            item.streams = self.get_qualities(labels)

        else:
            lgg.warning("Streams read error: %s" % error)

    def get_serial_collection(self, item, page):
        parser = SerialParser()
        parser.feed(page)

    def get_item_description(self, item):
        lgg.info("Search description from player")
        page = self.get_page_content(item.player)
        try:
            data = json.loads(page)
        except Exception as why:
            lgg.info("json read error: %s" % why)
            return

        try:
            items = data["data"]["attributes"]["metadata"]
            item.description = items["description"]
            item.duration = item.format_duration(items["duration"]["seconds"])
        except Exception as why:
            lgg.info("json keys error: %s" % why)

    def on_item_added(self, idx):
        """Verify the stream's urls for a video.

        Called when a video is added into the downloading basket.

        Args:
        idx -- index of the video
        """
        if self.is_in_category:
            video = self.ui.arte_list.video_items[idx].item

        else:
            video = self.videos[idx]

        if self.get_stream_urls(video):
            quality = self.core.cfg.get("tv_quality")
            video.set_file_name(self.core.cfg.get("add_date"))
            self.find_quality(video, quality)
            self.downloader.add_task(video)

        else:
            # Stream urls page not found
            self.ui.tv_basket.remove_item(video)
            #self.downloader.remove_task(video.idx)

    def get_stream_urls(self, item, page=None):
        url = item.player
        if url is None:
            url = self.build_url(item)

        if page is None:
            page = self.get_page_content(url)
        if page:
            streams, error = self.read_streams(page)
            if streams:
                labels = self.sort_streams_by_label(streams)
                item.streams = self.get_qualities(labels)
                if item.streams:
                    return True

                error = None

            if error is None:
                code, name = self.get_geolocation()
                error = "This video is not available, sorry."
                if code is not None:
                    if code not in ("DE, FR"):
                        error += "\nNote: you are located in %s" % name
                        if self.from_vpn(url, item):
                            return True

        else:
            error = "Can't get the page:\n%s" % url

        warn = WarningBox(5, error, self.ui)
        rep = warn.exec_()
        lgg.info("Streams list empty, display warning box")
        return False

    def from_vpn(self, page, item):
        #return False
        lgg.info("Trying by VPN ...")
        url = "https://www.oqapy.eu/arteservice?link=" + page
        try:
            txt = self.get_page_content(url)
        except Exception as why:
            lgg.info("VPN error: %s" % why)
            return False

        if txt:
            streams, error = self.read_streams(txt)
            if streams:
                labels = self.sort_streams_by_label(streams)
                item.streams = self.get_qualities(labels)
                if item.streams:
                    return True

        return False

    def read_streams(self, content):
        """Un-json the list of video's streams.

        Args:
        content -- str(web page)
        """
        try:
            dct = json.loads(content)
            # For debugging only
            #self.save_json(dct, "STREAMS")
            return dct["data"]["attributes"]["streams"], None
        except Exception as why:
            lgg.info("Can't read the stream urls: %s" % why)
            reason = "Can't read the stream urls: %s" % why

        return False, reason

    def sort_streams_by_label(self, data):
        streams = []
        for d in data:
            s = {"label": d["versions"][0]["label"], "url": d["url"]}
            streams.append(s)

        return streams

    def get_qualities(self, lst):
        number = 0
        for s in lst:
            versions = self.get_versions(s["url"])
            if versions:
                number += len(versions)

        if not number:
            return False

        return self.sort_streams_by_size(versions)

    def sort_streams_by_size(self, qualities):
        qualities[0] = sorted(qualities[0], 
                                        key=lambda item: item["resolution"][0], 
                                        reverse=True)

        return qualities

    def find_quality(self, item, expected=720):
        """Select the quality of a video.

        Args:
        item -- TVItem instance
        """

        for quality in item.streams[0]:
            if quality["resolution"][1] == expected:
                item.quality = quality

        if item.quality is None:
            lgg.warning("Quality %s expected not found!" % expected)
            item.quality = item.streams[0][0]

        for audio in item.streams[1]:
            if audio["code"] == self.lang:
                item.audio = audio
                item.subtitle = False

        if item.audio is None:
            item.audio = item.streams[1][0]
            try:
                item.subtitle = item.streams[2][0]
            except IndexError:
                # No subtitle i.e. concerts
                item.subtitle = None

            for s in item.streams[2]:
                if s["code"] == self.lang:
                    item.subtitle = s

    def get_versions(self, url):
        lgg.info("Get versions available\n\turl: %s" % url)
        if os.path.basename(url) == "master.m3u8":
            return self.get_hls_versions(url)

        qualities = []
        try:
            data = m3u8.load(url)
        except Exception as why:
            lgg.info("Can't find available version.\nReason: %s" % why)
            return []

        count = 0
        resos = []
        for playlist in data.playlists:
            qual = {}
            qual["id"] = count
            qual["resolution"] = playlist.stream_info.resolution
            if qual["resolution"] is None:
                continue

            qual["bandwidth"] = playlist.stream_info.bandwidth
            qual["uri"] = playlist.uri
            qual["base_url"] = data.base_uri
            qual["is_hls"] = False
            resos.append(qual)
            count += 1

        qualities.append(resos)
        try:
            audios = []
            subtitles = []
            medias = playlist.media
            for m in medias:
                if m.type == "AUDIO":
                    aud = {}
                    aud["code"] = m.language
                    aud["url"] = m.uri
                    aud["lang"] = m.name
                    audios.append(aud)

                elif m.type == "SUBTITLES":
                    sub = {}
                    sub["code"] = m.language
                    sub["url"] = m.uri
                    sub["lang"] = m.name
                    subtitles.append(sub)
        except Exception as why:
            lgg.info("Error in medias: %s" % why)
            return []

        qualities.append(audios)
        qualities.append(subtitles)
        return qualities

    def get_hls_versions(self, url):
        lgg.info("Get versions available for protocol hls")
        qualities = []
        try:
            data = m3u8.load(url)
        except Exception as why:
            lgg.warning("Can't load master: %s" % why)
            return qualities

        count = 0
        for playlist in data.playlists:
            qual = {"audio": None, "subtitle": None, "name": None, 
                    "is_hls": True, "lang": None}
            qual["id"] = count
            qual["resolution"] = playlist.stream_info.resolution
            qual["uri"] = playlist.absolute_uri
            qual["lang"] = self.find_language(playlist.absolute_uri)
            qualities.append(qual)

        return qualities

    def find_language(self, link):
        if "Q_0_VF" in link:
            return "VF Français"
        elif "Q_0_VOF" in link:
            return "VOF Français"
        elif "Q_0_VFAUD" in link:
            return "Français"
        elif "Q_0_VA" in link:
            return "VA Deutsch"
        elif "Q_0_VOA" in link:
            return "VOA Deutsch"
        elif "Q_0_VAAUD" in link:
            return "Deutsch"
        return "Original version"

    def build_url(self, item):
        base = 'https://api.arte.tv/api/player/v2/config'
        return '%s/%s/%s' %(base, self.lang, item.idx)

    def format_duration(self, value):
        if value is not None:
            min_, sec = divmod(value, 60)
            if min_ < 60:
                return "%s min. %s sec." %(min_, sec)

            hours, min_ = divmod(min_, 60)
            return "%s h. %s min. %s sec." %(hours, min_, sec)

    def get_page_content(self, url, header=None):
        lgg.info('Load page: %s' % url)
        # XXX bug 2102036 don't use a user-agent
        user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0)'\
                      ' Gecko/20100101 Firefox/49.0'
        if header is not None:
            header["User-Agent"] = user_agent

        else:
            header = {"User-Agent": user_agent}

        req = urllib.request.Request(url, data=None) # headers=header)
        
        try:
            content = urllib.request.urlopen(req, timeout=10)
            return str(content.read().decode('utf-8', 'replace'))
        except Exception as why:
            lgg.info('HTTP error (1): %s, %s' % (url, why))
            if "#" in url:
                url = url.replace("#", "/#")
                req = urllib.request.Request(url, data=None)
                lgg.info("Try with: %s" % url)
                try:
                    content = urllib.request.urlopen(req)
                    return str(content.read().decode('utf-8', 'replace'))
                except Exception as why:
                    lgg.info('HTTP error (2): %s, %s' % (url, why))

        return False

    def change_category(self, name):
        lgg.info("Change category: %s" % name)
        self.current_category = name

        if name == "arte+7":
            self.remove_filter()
            self.is_in_category = False

        elif name in self.categories_videos:
            category = self.categories_videos[name]
            previews = self.check_category_thumbnails(category)
            selection = []
            try:
                summary = [i for i in self.categories if i[0] == name]
                self.edit_category_data(summary[0][0], "", summary[0][1])
            except:
                self.edit_category_data(name, "", "")

            for video in category:
                item = self.build_item(video)
                if item:
                    selection.append(TVItem(item))

            for s in selection:
                s.pixmap = previews.pop(0)

            self.is_in_category = True
            self.ui.arte_list.display_category(selection)
            self.current_category = name

    def reload_category(self):
        self.change_category(self.current_category)

    def edit_category_data(self, title, subtitle, teaser):
        if not title.endswith("\n"):
            title += "\n"

        if subtitle is not None:
            if not subtitle.endswith("\n"):
                subtitle += "\n"

        else:
            subtitle = ""
        self.ui.pitch_editor.clear()
        font = QFont("sans", 10)
        c_form = QTextCharFormat()
        c_form.setFont(font)
        font1 = QFont("sans", 12)
        font1.setWeight(75)
        c_form1 = QTextCharFormat()
        c_form1.setFont(font1)
        self.ui.pitch_editor.setCurrentCharFormat(c_form1)
        self.ui.pitch_editor.appendPlainText(title)
        self.ui.pitch_editor.setCurrentCharFormat(c_form)
        self.ui.pitch_editor.insertPlainText(subtitle)
        if "</" in teaser:
            self.ui.pitch_editor.appendHtml("\n    %s" % teaser)

        else:
            self.ui.pitch_editor.appendPlainText("\n    %s" % teaser)

        self.ui.pitch_editor.verticalScrollBar().setValue(0)
        QCoreApplication.processEvents()

    def build_item(self, item):
        video = {}
        video["title"] = item["title"]
        if "programId" in item:
            video["id"] = item["programId"]

        else:
            parts = item["href"].split("/")
            try:
                ind = parts.index("videos")
                video["id"] = parts[ind+1]
            except ValueError:
                # bad object, rejected
                return False

        video["description"] = item.get("description", "")
        video["scheduled_on"] = None
        video["rights_end"] = None
        video["duration"] = item.get("duration", "")
        video["url"] = item["href"]
        video["subtitle"] = ""
        video["adult"] = ""
        video["thumbnail_url"] = item.get("src", "")
        video["views"] = ""
        return video

    def load_image(self, item):
        """Get the image file for the thumbnail.

        Args:
        item -- video item instance
        """
        ext = os.path.splitext(item.pix_url)[1] or ".jpg"
        path = os.path.join(self.cache, "%s%s" % (item.idx, ext))
        for i in range(2):
            try:
                with open(path, 'wb') as objfile:
                    f = urllib.request.urlopen(item.pix_url, timeout=5)
                    objfile.write(f.read())

                return path
            except Exception as why:
                lgg.info("Can't load image: %s" % item.pix_url)
                lgg.info("Reason: %s" % why)

        return "medias/noPreview.png"

    def check_category_thumbnails(self, cat):
        """Verify if all thumbnails exists.

        """
        lgg.info('Check thumbnails')
        thumbs = []
        for item in cat:
            orig = item["src"]
            if orig is not None:
                elems = orig.split("/")
                if orig.endswith((".jpg", ".jpeg", ".JPG", ".JPEG")):
                    fname = elems[-1]

                elif "x" in elems[-1]:
                    fname = elems[-2] + ".jpeg"

                else:
                    # Use 'No preview'
                    thumbs.append("medias/noPreview.png")
                    continue

                image = os.path.join(self.cache, fname)
                if not os.path.isfile(image):
                    image = self.download_preview(orig, image)

                thumbs.append(image)
            else:
                thumbs.append("medias/noPreview.png")

        return thumbs

    def display_category_content(self, idx):
        cat = self.ui.arte_list.video_items[idx]
        item = cat.item
        lgg.info("Choose: %s, id: %s" % (item.title, item.idx))
        if not item.idx.startswith("RC"):
            self.edit_summary(item)

        else:
            self.display_sub_category(item)
            self.ui.refresh_btn.setEnabled(True)

    def display_sub_category(self, item):
        lgg.info("display_sub_category")
        url = "https://api.arte.tv/api/player/v2/config/%s/%s" %(self.lang, item.idx)
        content = self.get_page_content(url)
        if not content:
            lgg.info("No response from %s" % url)
            return

        try:
            data = json.loads(content)
        except Exception as why:
            lgg.warning("Read json error: %s" % why)
            return

        try:
            link = data["data"]["attributes"]["metadata"]["link"]["url"]
        except Exception as why:
            lgg.warning("Key error: %s" % why)
            return

        parts = link.split("/")
        ind, name = parts[-3:-1]
        parts2 = name.split("-")
        number = int(parts2[-1])
        parts3 = ind.split("-")
        videos = []
        for i in range(1, number+1):
            parts2[-2] = str(i)
            vid = "-".join(parts2)
            idx = "{}-{:0>3}-A".format(parts3[0], str(i))
            d = {}
            d["programId"] = idx
            d["title"] = vid
            d["description"] = item.description
            d["duration"] = item.duration
            d["href"] = "https://api.arte.tv/api/player/v2/config/%s/%s" %(
                                                            self.lang, idx)
            d["thumbnail_url"] = ""
            videos.append(d)

        selection = []
        for video in videos:
            nitem = self.build_item(video)
            if nitem:
                selection.append(TVItem(nitem))
                selection[-1].pixmap = item.pixmap

        self.is_in_category = True
        self.ui.arte_list.display_category(selection)

    def display_sub_category2(self, category):
        url = category.item.url
        lgg.info("Sub category: %s" % url)
        content = self.get_page_content(url)
        if not content:
            lgg.info("No response from %s" % url)

        else:
            name, comment, desc, videos = self.get_sub_category_items(content)
            if not videos:
                lgg.warning("No videos in subcategory %s" % url)
                return

            selection = []
            for video in videos:
                item = self.build_item(video)
                if item:
                    selection.append(TVItem(item))
                    selection[-1].pixmap = category.item.pixmap

            self.is_in_category = True
            self.ui.arte_list.display_category(selection)
            self.edit_category_data(name, comment, desc)

    def get_sub_category_items(self, page):
        videos = []
        title, subtitle, desc = "*", "**", "***"
        try:
            data = page.split('<script id="__NEXT_DATA__" type="application/json">')[1]
            data = data.split("</script>")[0].rstrip(" \n;")

            data = json.loads(data)
            self.zones = []
            self.get_zones(data)
            # Debug usage
            #with open("example", "w") as outf:
                #jsn = json.dumps(self.zones, sort_keys=False, indent=2, 
                        #separators=(',', ': '), ensure_ascii=False)
                #outf.write(jsn)
            d = self.zones[0]
            title = d["title"]
            subtitle = d["title"]
            desc = d["content"]["data"][0]["shortDescription"]
            for z in self.zones:
                items = z["content"]["data"]
                if len(items) > 1:
                    for i in items:
                        try:
                            d = {}
                            d["programId"] = i["id"]
                            d["title"] = i["title"]
                            d["description"] = i.get("shortDescription", "")
                            d["duration"] = i.get("duration", "")
                            d["href"] = i["url"]
                            d["src"] = self.choose_thumbnail("")
                            videos.append(d)
                        except Exception as why:
                            continue
        except Exception as why:
            lgg.warning("Load subcategory error: %s" % why)

        return title, subtitle, desc, videos

    def choose_thumbnail(self, imgs):
        img = None
        try:
            if "landscape" in imgs:
                img = imgs["landscape"]["resolutions"][0]["url"]

            elif "square" in imgs:
                img = imgs["square"]["resolutions"][0]["url"]
        except:
            pass
        return img

    def start_download(self):
        self.ui.tv_basket_buttons.on_progress()
        self.ui.action_Cancel.setEnabled(True)
        self.downloader.no_merging = False
        self.downloader.start_downloading()

    def cancel_download(self):
        lgg.info("Cancel downloading")
        self.downloader.no_merging = True
        self.downloader.abort_task()
        self.ui.tv_basket_buttons.clear()
        self.ui.action_Cancel.setEnabled(False)
        if self.ui.tv_basket.lst_movies:
            self.ui.tv_basket_buttons.dwnld_btn.setEnabled(True)

    def set_filter(self, key=None):
        if key is None:
            key = self.ui.filter_led.text()

        if not key:
            return

        keys = [k.lower() for k in key.split(' ')]
        selected = []
        for idx, v in enumerate(self.videos):
            title = v.title.lower()
            for k in keys:
                if k in title:
                    selected.append((idx, v))
                    break

        if selected:
            self.ui.arte_list.display_filtered_videos(selected)
            self.ui.showall_btn.setEnabled(True)

        self.update_filter_label(len(selected))
        self.update_filter_keys(key)

    def remove_filter(self):
        self.update_filter_label()
        self.ui.showall_btn.setEnabled(False)
        self.ui.filter_led.blockSignals(True)
        self.ui.filter_led.setText('')
        self.ui.filter_led.setInformativeText(_('Enter a keyword'))
        self.ui.filter_led.blockSignals(False)
        QCoreApplication.processEvents()
        self.ui.arte_list.display_videos()

    def update_filter_label(self, num=None):
        if num is None:
            self.ui.result_lbl.hide()
            return

        self.ui.result_lbl.show()
        if num == 0:
            text = 'No such videos'

        elif num == 1:
            text = 'One video found'

        else:
            text = '{0} videos found'.format(str(num))

        self.ui.result_lbl.setText(text)

    def update_filter_keys(self, new):
        keys = self.core.cfg.get("filter_keys")
        if new in keys:
            keys.remove(new)

        keys.insert(0, new)
        self.core.cfg.update("filter_keys", keys[:10])

    def resize_thumbnails(self, size):
        if self.core.cfg.get('thumb1') != size:
            self.ui.arte_list.setIconSize(QSize(size, size))
            self.core.cfg.update('thumb1', size)

    def get_video_dir(self):
        return self.core.cfg.get("videos_folder")

    def merge_files(self):
        return self.core.cfg.get("merge_files")

    def save_json(self, data, fname):
        """Debugging function

        """
        with open(fname, "w") as outf:
            jsn = json.dumps(data, sort_keys=False, indent=2, 
            separators=(',', ': '), ensure_ascii=False)
            outf.write(jsn)

    def configure_downloading(self, video):
        """Call the downloading config dialog box.

        Args:
        video -- TVItem instance
        """
        lgg.info('Configure downloading of %s' % video.title)
        self.cfg = TvDownloadingConfig(self)
        self.cfg.configure(video, self.core.cfg.get('tv_quality'),
                           self.core.cfg.get('lang'), 
                           date=self.core.cfg.get('add_date'),
                           subtitle=self.core.cfg.get('add_subtitle'))
        self.cfg.exec_()

    def set_preferred_quality(self, quality):
        self.core.cfg.update('tv_quality', quality)

    def display_progress(self, percent, speed, remain):
        """Show the downloading progress informations.

        Args:
        percent -- percent downloaded
        speed -- bandwidth in Ko/sec.
        remain -- estimated remaining time, 0 for hls protocol
        """
        sp = self.locale.toString(int(speed / 1024))
        self.ui.tv_basket_buttons.progressBar.setValue(percent)
        self.ui.tv_basket_buttons.progress_lbl.setText(
                                "Speed: %s Kio/sec." % sp)
        if remain:
            # In hls protocol the remaining time is not computable
            rtime = int(remain / speed)
            m, s = divmod(rtime, 60)
            self.ui.tv_basket_buttons.progress_lbl2.setText(
                                "Remaining time: %s min. %s sec." %(m, s))

    def on_downloading_finished(self, fname, remain):
        """Rename the video downloaded.

        Args:
        fname -- temporary file name
        """
        lgg.info('File %s downloaded' % fname)
        #item = self.ui.tv_basket.remove_item(0)
        self.ui.tv_basket_buttons.clear(remain)
        self.ui.action_Cancel.setEnabled(False)
        if remain:
            self.ui.action_Cancel.setEnabled(True)
            self.ui.tv_basket_buttons.cancel_btn.setEnabled(True)

    def on_merging_finished(self, video, fname):
        lgg.info("Merging finished: %s" % fname)
        self.copy_summary(video, fname)
        self.copy_thumbnail(video, fname)
        if not self.downloader.is_running:
            self.ui.tv_basket_buttons.clear()
            if not self.is_pending_tasks():
                self.ui.tv_basket_buttons.progress_lbl2.setText(
                                    "All tasks finished.")

    def on_empty_list(self):
        if self.downloader.is_merging:
            self.ui.tv_basket_buttons.progress_lbl2.setText(
                                    "Merging latest video ...")

        else:
            self.ui.tv_basket_buttons.clear()

    def download_preview(self, url, fname):
        for i in range(2):
            try:
                with open(fname, 'wb') as objfile:
                    f = urllib.request.urlopen(url, timeout=5)
                    objfile.write(f.read())

                return fname
            except Exception as why:
                lgg.info("Can't load image: %s" % url)
                lgg.info("Reason: %s" % why)

        return "medias/noPreview.png"

    def copy_summary(self, video, fname):
        """Save the summary into a file.

        Args:
        video -- TVItem instance
        fname -- path-name of the video downloaded
        """
        if not self.core.cfg.get('pitch_plus'):
            return

        if self.core.cfg.get('pitch_plus_unique'):
            path = os.path.join(os.path.dirname(fname), 'index')

        else:
            path = os.path.splitext(fname)[0]

        try:
            with open(path, 'a') as outf:
                outf.write(video.summary)
                lgg.info("Summary saved")
        except Exception as why:
            lgg.warning("Can't write summary: %s" % path)
            lgg.warning("Reason: %s" % why)

    def copy_thumbnail(self, video, fname):
        """Save the thumbnail into a file.

        Args:
        video -- TVItem instance
        fname -- path-name of the video downloaded
        """
        if self.core.cfg.get('copythumb_plus'):
            path = os.path.splitext(fname)[0] + ".jpg"
            orig = video.pixmap
            shutil.copy(orig, path)
            lgg.info("Thumbnail saved")

    def get_geolocation(self):
        url = "https://www.arte.tv/artews/services/geolocation.xml"
        head = {"Accept": "application/json, text/plain, */*"}
        content = self.get_page_content(url, header=head)
        if "<?xml " in content:
            return self.read_xml(content)

        try:
            data = json.loads(content)
            return data["countryCode"], data["countryName"]
        except Exception as why:
            lgg.info("Can't read geolocation: %s" % why)
            return None, None

    def read_xml(self, xml):
        lines = xml.split("\n")
        for l in lines:
            if "<countryCode>" in l:
                code = l.split(">")[1].split("<")[0]
            elif "<countryName>" in l:
                name = l.split(">")[1].split("<")[0]
                return code, name

        return None, None

    def is_pending_tasks(self):
        if self.downloader.is_running or self.downloader.is_merging:
            return True

        return False

    def handle_downloading_error(self, loader):
        """Display a warning box when the downloading failed.

        """
        url = loader.url
        error = loader.last_error
        lgg.warning("Downloading error: %s" % error)
        lgg.warning("URL: %s" % url)
        warn = WarningBox(8, (url, error), self.ui)
        rep = warn.exec_()
        self.ui.tv_basket.remove_item(0)
        if self.ui.tv_basket.lst_movies:
            self.start_download()

class HtmlParser(HTMLParser):
    def __init__(self, url, lang):
        super().__init__()
        self.url = url
        self.lang = lang
        self.videos = []
        self.indexes = []
        diff = timedelta(days=10)
        last = date.today() - diff
        now = last.strftime("%Y%m%d")
        self.day_before = self.get_day_before(last)
        self.in_section = False
        self.in_item = False
        self.in_image = False
        self.item = {}
        self.in_title = False
        self.in_title1 = False

    def get_next_page(self):
        url = self.url.replace("DAY", next(self.day_before))
        page = self.fetch_page(url)
        return page
        if not page:
            return []

        self.feed(page)
        return self.videos

    def get_day_before(self, start):
        now = start
        diff = timedelta(days=1)
        while 1:
            now = now + diff
            yield now.strftime("%Y%m%d")

    def fetch_page(self, url):
        lgg.info('Fetch page: %s' % url)
        req = urllib.request.Request(url, data=None)
        try:
            content = urllib.request.urlopen(req)
        except Exception as why:
            lgg.warning('urllib error: %s, %s' % (url, why))
            return False

        return str(content.read().decode('utf-8', 'replace'))

    def handle_starttag(self, tag, attrs):
        if tag == "div":
            for attr in attrs:
                #if attr == ("class", "ds-19ooqlf"):
                if attr == ("data-testid", "ts-tsItem"):
                    self.in_section = True

        elif self.in_section and tag == "a":
            for attr in attrs:
                if attr == ("class", "ds-1lzc6v3"):
                    self.in_item = True
                    self.item = {}

                elif self.in_item and attr[0] == "href":
                    self.item["url"] = attr[1]
                    self.in_image = True

        elif self.in_image and tag == "img":
            for attr in attrs:
                if attr[0] == "src":
                    self.item["thumbnail_url"] = attr[1]
                    self.in_title = True

        elif tag == "h3" and self.in_title:
            self.in_title1 = True

    def handle_data(self, data):
        if self.in_title1:
            self.item["title"] = data
            self.in_title = False
            self.in_title1 = False
            self.in_item = False
            self.in_image = False
            self.videos.append(self.item)

class SerialParser(HTMLParser):
    """Parser general"""
    def __init__(self):
        super().__init__()
        self.videos = []
        self.in_item = False
        self.in_title = False
        self.in_title1 = False
        self.in_desc = False

    def handle_starttag(self, tag, attrs):
        if tag == "div":
               for attr in attrs:
                    if attr == ("data-testid", "ts-tsItem"):
                        self.in_item = True
                        self.item = {}

        if self.in_item:
            if tag == "a":
               for attr in attrs:
                    if attr[0] == "href":
                        self.item["href"] = attr[1]

            if tag == "img":
                for attr in attrs:
                    if attr[0] == "src":
                        self.item["src"] = attr[1]

            if tag == "div":
                for attr in attrs:
                    if attr == ("class", "ds-1h05j9g"):
                        self.in_title = True

        if self.in_title:
            if tag == "h3":
                self.in_title1 = True

            elif tag == "p":
                self.in_desc = True

    def handle_data(self, data):
        if self.in_title1:
            self.item["title"] = data
            self.in_title1 = False

        elif self.in_desc:
            self.item["description"] = data
            self.in_desc = False
            self.in_title = False
            self.in_item = False
            self.videos.append(self.item)

class ItemParser(HTMLParser):
    """Parser general"""
    def __init__(self):
        super().__init__()
        self.in_duration = False
        self.in_desc = False
        self.in_teaser = False
        self.in_cast = False
        self.in_label = False
        self.in_cast1 = False
        self.in_actor = False
        self.in_real = False
        self.in_real1 = False
        self.in_orig = False
        self.in_orig1 = False
        self.in_date = False
        self.in_date1 = False
        self.item = {"desc": "", "cast": [], "real": [],
                     "country": [], "date": None, "duration": ""}

    def handle_starttag(self, tag, attrs):
        if tag == "div":
            for attr in attrs:
                if attr == ("class", "ds-66uxch"):
                    self.in_duration = True
                elif attr == ("class", "ds-t47jm3"):
                    self.in_desc = True
                elif attr == ("class", "ds-qcz0i0"):
                    self.in_teaser = True
                elif attr == ("class", "ds-1i28n7i"):
                    self.in_desc = False
                    self.in_cast = True

        elif tag == "p":
            for attr in attrs:
                if attr == ("class", "ds-1rjvccl"):
                    self.in_label = True
                elif attr == ("class", "ds-ykt0na"):
                    if self.in_cast1:
                        self.in_actor = True
                    elif self.in_real:
                        self.in_real1 = True
                        self.in_actor = False
                    elif self.in_orig:
                        self.in_orig1 = True
                        self.in_real = False
                        self.in_real1 = False
                    elif self.in_date:
                        self.in_orig1 = False
                        self.in_date1 = True

    def handle_data(self, data):
        if self.in_duration:
            self.item["duration"] = data
            self.in_duration = False
        elif self.in_label:
            if data in ("Avec", "Mit"):
                self.in_cast1 = True
                self.in_label = False
            elif data in ("Réalisation", "Regie"):
                self.in_real = True
                self.in_cast1 = False
                self.in_label = False
            elif data in ("Pays", "Land"):
                self.in_orig = True
                self.in_real = False
                self.in_real1 = False
                self.in_label = False
            elif data in ("Année", "Jahr"):
                self.in_date = True
                self.in_real1 = False
                self.in_orig = False
                self.in_label = False
            else:
                self.in_actor = False
                self.in_real1 = False
                self.in_orig1 = False
                self.in_date1 = False

        elif self.in_desc:
            self.item["desc"] += data
        elif self.in_teaser:
            self.item["teaser"] = data
            self.in_teaser = False
        elif self.in_actor:
            self.in_actor = False
            self.item["cast"].append(data)
        elif self.in_real1:
            self.item["real"].append(data)
            self.in_real1 = False
        elif self.in_orig1:
            self.item["country"].append(data)
            self.in_orig1 = False
        elif self.in_date1:
            self.item["date"] = data
            self.in_date = False
            self.in_date1 = False


class TVItem:
    """Define a video item.

    """
    def __init__(self, attr, lang='fr'):
        self.idx = attr['id']
        self.title = attr['title']
        # Very short description
        self.teaser = attr.get('teaser', '')
        self.sched = attr.get('scheduled_on', None)
        self.date = attr.get('scheduled_on', None)
        self.duration = self.format_duration(attr.get('duration', None))
        self.url = attr['url']
        self.subtitle = attr.get('subtitle', "")
        self.pix_url = attr['thumbnail_url']
        self.pixmap = None
        # description, origin and anno must be extracted from the video page
        self.shortdesc = attr.get('shortdesc', '')
        self.description = attr.get('description', '')
        self.header = attr.get('headerText', '')
        self.origin = ''
        self.anno = ''
        self.player = None
        self.streams = None
        self.quality = None
        self.audio = None
        self.lang = lang
        self.subtitle = None
        self.label = '' # if '' the page was not yet parsed
        self.realisation = '' # list
        self.music = '' # list
        self.actors = ''
        self.summary = ''
        self.set_file_name(False)
        if self.date is not None:
            if not 'T' in self.date:
                self.date = self.format_date(attr['scheduled_on'], 
                                             attr['rights_end'])
            else:
                # Remove the time zone '+02'
                d = self.date.split('+')[0].replace('T', ' ').lstrip("'")
                self.date = d.replace(":", "-")
        
    def format_date(self, begin, end):
        """Return the release date and time of the video.

        """
        # The release value contents only the date, we take the expiration
        # time and we add two hours
        if not end:
            if not begin:
                return "No date"

            end = begin

        end = end.split('T')[1].rstrip('Z')
        h, m, s = end.split(':')
        h = str((int(h) + 2) % 24)
        return "%s %s" %(begin, '-'.join([h, m, s]))

    def format_duration(self, value):
        if isinstance(value, str):
            return value

        if value is not None:
            min_, sec = divmod(value, 60)
            if min_ < 60:
                return "%s min. %s sec." %(min_, sec)

            hours, min_ = divmod(min_, 60)
            return "%s h. %s min. %s sec." %(hours, min_, sec)

    def get_pixmap(self):
        title = self.title.lower()
        if 'xenius' in title:
            return 'medias/Xenius.jpg'

        if 'journal' in title:
            if 'best' in title:
                return 'medias/best_journal.jpg'
            if 'junior' in title:
                return 'medias/journal_junior.jpg'
            return 'medias/journal.jpg'

        if 'dessous des cartes' in title or 'mit offenen karten' in title:
            return 'medias/dessous_cartes.jpg'

    def set_file_name(self, date):
        f = self.title.replace('/', '-').replace('»', '_').replace('«', '_')
        self.fname = f.replace('\\', ' ').replace('"', '')
        if date and self.date is not None:
            self.fname += '_%s' % self.date.replace(' ', '_')

