Tomo Kihara
tiktoktoad

Repository

$ git clone https://gitlab.waag.org/lodewijk/tiktoktoad.git
# tested with node v16.13.2, npm 8.1.2
$ sudo npm install --global web-ext
$ gem install sequel
$ gem install sqlite3
$ gem install sinatra
# in a seperate terminal tab or window
tiktoktoad/webservice$ ruby log_service.rb
tiktoktoad/webext$ web-ext run --start-url https://www.tiktok.com/
tiktoktoad/webservice$ sqlite3 tiktok.db

# check for ENABLE_JSON1
sqlite> PRAGMA compile_options;
# get music urls from video json data
sqlite> select json_extract(video.data, '$.music.playUrl') from video;
# get video download url from video json data
sqlite> select json_extract(video.data, '$.video.downloadAddr') from video;
# get video by specific json data property
sqlite> select description from video where json_extract(video.data, '$.author.nickname') = 'Kika Kim';
# all videos with their label
sqlite> select label, video.description from label join video_label on label.id = video_label.label_id join video on video_label.video_id = video.id;
# find most recommend videos in all sessions
sqlite> select video.description, video.id, count(video_id) as recommends from recommend join video on recommend.video_id = video.id group by video_id order by recommends DESC;
# get events and specific data
sqlite> select name, timestamp, json_extract(event.data, '$.params.page_name') from event;
sqlite> select name, timestamp, json_extract(event.data, '$.params.duration') from event;
# find video's with a certain tag
sqlite> select video.id, video.description, json_extract(video.data, '$.video.downloadAddr') label from label join video_label on video_label.label_id = label.id join video on video_label.video_id = video.id where label = "Scripted Comedy";
#
sqlite> select timestamp, video.data from recommend join video on recommend.video_id = video.id order by timestamp desc limit 10;
# check most used labels
sqlite> select label, count(label) as count from label join video_label on label.id = video_label.label_id join video on video_label.video_id = video.id group by label order by count DESC ;
about:debugging#/runtime/this-firefox
                                                      ┌────────────┐  ┌────────────┐
                                                      │video_tag   │  │tag         │
                                                      │------------│  │------------│
                                                      │tag         │x─┤id          │
                                                   ┌─x│video       │  │tag         │
                                                   │  │            │  │            │
                                                   │  └────────────┘  └────────────┘
                                                   │
                                                   │  ┌────────────┐  ┌────────────┐
                                                   │  │video_label │  │label       │
                                                   │  │------------│  │------------│
                  ┌────────────┐                   │  │label       │x─┤id          │
                  │event       │                   ├─x│video       │  │label       │
                  │------------│    ┌────────────┐ │  │            │  │            │
                  │id          │    │video       │ │  └────────────┘  └────────────┘
                  │name        │    │------------│ │
                  │video       │x─┬─┤id          ├─┘  ┌────────────┐
                  │...         │  │ │title       │    │music       │
┌────────────┐    │timestamp   │  │ │description │    │------------│
│session     │ ┌─x│session     │  │ │music       │x───┤id          │
│------------│ │  └────────────┘  │ │...         │    │title       │
│id          ├─┤                  │ │author      │x─┐ │...         │
│video       │ │                  │ └────────────┘  │ │            │
│...         │ │  ┌────────────┐  │                 │ │            │
│timestamp   │ │  │recommend   │  │                 │ └────────────┘
│            │ │  │------------│  │                 │
│            │ │  │id          │  │                 │ ┌────────────┐
└────────────┘ │  │video       │x─┘                 │ │author      │
               │  │...         │                    │ │------------│
               │  │timestamp   │                    └─┤id          │
               ├─x│session     │                      │unique_name │
               │  │            │                      │nick        │
               │  └────────────┘                      │...         │
               │                                      │            │
               │  ┌────────────┐                      │            │
               │  │session_tag │                      └────────────┘
               │  │------------│
               │  │id          │
               │  │tag         │
               │  │timestamp   │
               └─x│session     │
                  │            │
                  └────────────┘
{
  "event": "play_time",
  "params": "{\"page_name\":\"homepage_hot\",\"enter_from\":\"homepage_hot\",\"is_landing_page\":1,\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:96.0) Gecko/20100101 Firefox/96.0\",\"region\":\"NL\",\"page_url\":\"https://www.tiktok.com/\",\"group_id\":\"7051840377233476870\",\"author_id\":\"6944934559298618373\",\"play_mode\":\"one_column\",\"autoplay_status\":1,\"duration\":3643,\"enter_method\":\"\",\"search_id\":\"\",\"focus_time\":3327,\"is_from_webapp\":\"0\",\"event_index\":1643807729616}",
  "local_time_ms": 1643807314056,
  "is_bav": 0,
  "ab_sdk_version": "",
  "session_id": "9f1cdce1-6de1-4c0f-9216-51d41ffc87ab"
}
{
  "id": "7048695864558161158",
  "desc": "Besef dit was gewoon een jaar geleden 😂  #fyp #foryou",
  "createTime": "1641152396",
  "scheduleTime": 0,
  "video": {
    "id": "7048695864558161158",
    "height": 1024,
    "width": 576,
    "duration": 30,
    "ratio": "720p",
    "cover": "https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/218ee74c5ede481d887495a40ffb8f90_1641152398?x-expires=1643828400&x-signature=OSvgHp9EjBCmRtg4tD5wks1UHus%3D",
    "originCover": "https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/c9a1078161b8449e92ba53cb962b3c5c_1641152397?x-expires=1643828400&x-signature=jSw9sdS1g2Ak%2Fo448Rk9hRmCt5s%3D",
    "dynamicCover": "https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/675e1c4d84214fe7986a617a7a06f217_1641152397?x-expires=1643828400&x-signature=dKrajtemZfop7BafbCVdqDB0OAw%3D",
    "playAddr": "https://v16-webapp.tiktok.com/33afe9df5841517acb60ae47014332db/61fad6c8/video/tos/useast2a/tos-useast2a-ve-0068c004/3b60bc3c24ba4212b487ddb1aecace04/?a=1988&br=3046&bt=1523&cd=0%7C0%7C1%7C0&ch=0&cr=0&cs=0&cv=1&dr=0&ds=3&er=&ft=XOQ9-3tZnz7ThNjL.lXq&l=202202021308250102231230341A53D896&lr=tiktok_m&mime_type=video_mp4&net=0&pl=0&qs=0&rc=amhoNGg6ZnV5OjMzNzczM0ApOGk7NDw2Ojs7NzZkaTVmZ2dwNW9ocjRvcjVgLS1kMTZzc2JfYmA0M2BiYTY2My01MWA6Yw%3D%3D&vl=&vr=",
    "downloadAddr": "https://v16-webapp.tiktok.com/33afe9df5841517acb60ae47014332db/61fad6c8/video/tos/useast2a/tos-useast2a-ve-0068c004/3b60bc3c24ba4212b487ddb1aecace04/?a=1988&br=3046&bt=1523&cd=0%7C0%7C1%7C0&ch=0&cr=0&cs=0&cv=1&dr=0&ds=3&er=&ft=XOQ9-3tZnz7ThNjL.lXq&l=202202021308250102231230341A53D896&lr=tiktok_m&mime_type=video_mp4&net=0&pl=0&qs=0&rc=amhoNGg6ZnV5OjMzNzczM0ApOGk7NDw2Ojs7NzZkaTVmZ2dwNW9ocjRvcjVgLS1kMTZzc2JfYmA0M2BiYTY2My01MWA6Yw%3D%3D&vl=&vr=",
    "shareCover": [
      "",
      "https://p16-sign-va.tiktokcdn.com/tos-maliva-p-0068/c9a1078161b8449e92ba53cb962b3c5c_1641152397~tplv-tiktok-play.jpeg?x-expires=1644411600&x-signature=qc%2BEGGGx5Y%2FoIUKaZeqo7ymS4Gc%3D",
      "https://p16-sign-va.tiktokcdn.com/tos-maliva-p-0068/c9a1078161b8449e92ba53cb962b3c5c_1641152397~tplv-tiktokx-share-play.jpeg?x-expires=1644411600&x-signature=QaFUzRwde6CQ0%2BCDX4oFC4bJuCg%3D"
    ],
    "reflowCover": "https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/af9bd9723de38f0199e20f9aaa5807f3?x-expires=1643828400&x-signature=E8%2BuE8yxAtJS0ztdvdk%2BSZLVU58%3D",
    "bitrate": 1559564,
    "encodedType": "normal",
    "format": "mp4",
    "videoQuality": "normal",
    "encodeUserTag": "",
    "codecType": "h264",
    "definition": "720p"
  },
  "author": "iliasvietto17",
  "music": {
    "id": "7048695834539559685",
    "title": "origineel geluid",
    "playUrl": "https://sf77-ies-music-va.tiktokcdn.com/obj/musically-maliva-obj/7048695862213790469.mp3",
    "coverLarge": "https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/4caa1210488a1551c4804431171493ec~c5_1080x1080.jpeg?x-expires=1643893200&x-signature=yhgYrzGYcfKPRWTQ3AoWQZYQ9gc%3D",
    "coverMedium": "https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/4caa1210488a1551c4804431171493ec~c5_720x720.jpeg?x-expires=1643893200&x-signature=v%2B11cxHy0bpIBcUgBB9%2BrSQHwkk%3D",
    "coverThumb": "https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/4caa1210488a1551c4804431171493ec~c5_100x100.jpeg?x-expires=1643893200&x-signature=JfIIErCax2y83UXY0bvSI8DjMqA%3D",
    "authorName": "Viètto El Piètta",
    "original": true,
    "duration": 30,
    "album": "",
    "scheduleSearchTime": 0
  },
  "challenges": [
    {
      "id": "229207",
      "title": "fyp",
      "desc": "",
      "profileLarger": "",
      "profileMedium": "",
      "profileThumb": "",
      "coverLarger": "",
      "coverMedium": "",
      "coverThumb": "",
      "isCommerce": false
    },
    {
      "id": "42164",
      "title": "foryou",
      "desc": "",
      "profileLarger": "",
      "profileMedium": "",
      "profileThumb": "",
      "coverLarger": "",
      "coverMedium": "",
      "coverThumb": "",
      "isCommerce": false
    }
  ],
  "stats": {
    "diggCount": 83800,
    "shareCount": 1668,
    "commentCount": 2064,
    "playCount": 734900
  },
  "duetInfo": {
    "duetFromId": "0"
  },
  "warnInfo": [],
  "originalItem": false,
  "officalItem": false,
  "textExtra": [
    {
      "awemeId": "",
      "start": 42,
      "end": 46,
      "hashtagId": "229207",
      "hashtagName": "fyp",
      "type": 1,
      "subType": 0,
      "userId": "",
      "isCommerce": false,
      "userUniqueId": "",
      "secUid": ""
    },
    {
      "awemeId": "",
      "start": 47,
      "end": 54,
      "hashtagId": "42164",
      "hashtagName": "foryou",
      "type": 1,
      "subType": 0,
      "userId": "",
      "isCommerce": false,
      "userUniqueId": "",
      "secUid": ""
    }
  ],
  "secret": false,
  "forFriend": false,
  "digged": false,
  "itemCommentStatus": 0,
  "showNotPass": false,
  "vl1": false,
  "takeDown": 0,
  "itemMute": false,
  "effectStickers": [],
  "authorStats": {
    "followerCount": 601900,
    "followingCount": 194,
    "heart": 26800000,
    "heartCount": 26800000,
    "videoCount": 447,
    "diggCount": 18900
  },
  "privateItem": false,
  "duetEnabled": true,
  "stitchEnabled": true,
  "stickersOnItem": [],
  "isAd": false,
  "shareEnabled": true,
  "comments": [],
  "duetDisplay": 0,
  "stitchDisplay": 0,
  "indexEnabled": true,
  "diversificationLabels": [
    "Comedy",
    "Performance"
  ],
  "nickname": "Viètto El Piètta",
  "authorId": "6736668404306035718",
  "authorSecId": "MS4wLjABAAAAStO-uymL2x65GD8vrTc8cHnOkOWnhHqU1L9-BX_tuVTmugc6kk1j3Gghd-oA8YDB",
  "avatarThumb": "https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/4caa1210488a1551c4804431171493ec~c5_100x100.jpeg?x-expires=1643893200&x-signature=JfIIErCax2y83UXY0bvSI8DjMqA%3D",
  "batch": "1"
}
{
  "ftcUser": false,
  "secUid": "MS4wLjABAAAAC7jPWE5P0mAU4yBan6yaSr6bXpCJUf652omWJitiOrLdcZg1L-4hlQgE9J_cKjk_",
  "uid": "7048788831880922118",
  "nickName": "Rommel Zooi",
  "signature": "",
  "uniqueId": "rommelzooi",
  "createTime": "1641208395",
  "hasLivePermission": false,
  "roomId": "",
  "region": "",
  "avatarUri": ["https://p16-sign-va.tiktokcdn.com/musically-maliva-obj/70489…es=1641301200&x-signature=xtOG%2B33AKboit%2FCDKdbXPvbmHyA%3D"],
  "isPrivateAccount": false,
  "hasIMPermission": true,
  "showPrivateBanner": false,
  "showScheduleTips": false,
  "longVideoMinutes": 0,
  "ageGateRegion": "NL",
  "ageGateTime": "1641208394513",
  "userMode": 1,
  "proAccountInfo": {
    "status": 0,
    "analyticsOn": false,
    "businessSuiteEntrance": false,
    "downloadLink": {}
  },
  "analyticsOn": false,
  "redDot": []
}