Help Debugging MiniTV project
I was hoping I might be able to get help debugging my Mini-TV-ESP32 project.
I copied the Mini-TV-esp32 project on YouTube & GitHub, but I can't seem to get it working, and I was hoping someone might be able to help me out. I'm still very green on Arduinos and programming, but I thought I'd give this a try for a project.
I copied the files for the program into Arduino 2.3.8 and followed the video on YouTube. I've been able to test the program and compile and upload to my ESP32, but all that happens is I get a quick chirp from the speaker, and that's it. No activity on the screen or anything.
For hardware, I'm using the following parts, all sourced on Amazon
Here are some pictures of my test assembly
This is the main sketch for my file, it's called sketch_starfighter.ino. I followed the instructions in the youtube video to use ffmpeg to convert my video file into an mjpeg and aac file. I did make some tweaks b/c my video is 480x320 instead of the creators 288 x 240. I am also using the creators esp32_audio_task.h and mjpeg_decode_draw_task.h files, which I have also copied below
sketch_starfighter.ino
/***
* Required libraries:
* https://github.com/moononournation/Arduino_GFX.git
* https://github.com/pschatzmann/arduino-libhelix.git
* https://github.com/bitbank2/JPEGDEC.git
*/
// Audio and video code
// ffmpeg -i LSF_Reel.mp4 -ar 44100 -ac 1 -ab 24k -filter:a loudnorm -filter:a "volume=-5dB" LSF_Reel.aac
// ffmpeg -i LSF_Reel.mp4 -vf "fps=25,scale=-1:240:flags=lanczos,crop=320:in_h:(in_w-480)/2:0" -q:v 11 LSF_Reel.mjpeg
// auto fall back to MP3 if AAC file not available
#define AAC_FILENAME "/LSF_Reel.aac"
#define MP3_FILENAME "/LSF_Reel.mp3"
// #define MP3_FILENAME "/EP05.mp3"
#define MJPEG_FILENAME "/LSF_Reel.mjpeg"
// #define MJPEG_FILENAME "/EP05.mjpeg"
// #define MJPEG_FILENAME "/320_30fps.mjpeg"
#define FPS 25
#define MJPEG_BUFFER_SIZE (480 * 320 * 2 / 8)
// #define MJPEG_BUFFER_SIZE (320 * 240 * 2 / 8)
#define AUDIOASSIGNCORE 1
#define DECODEASSIGNCORE 0
#define DRAWASSIGNCORE 1
#include <WiFi.h>
#include <FS.h>
#include <LittleFS.h>
#include <SPIFFS.h>
#include <FFat.h>
#include <SD.h>
#include <SD_MMC.h>
/* Arduino_GFX */
#include <Arduino_GFX_Library.h>
#define GFX_BL DF_GFX_BL // default backlight pin, you may replace DF_GFX_BL to actual backlight pin
Arduino_DataBus *bus = create_default_Arduino_DataBus();
// Arduino_GFX *gfx = new Arduino_ILI9341(bus, DF_GFX_RST, 3 /* rotation */, false /* IPS */);
Arduino_GFX *gfx = new Arduino_ST7796(bus, DF_GFX_RST, 1 /* rotation */, true /* IPS */, 240 /* width */, 288 /* height */, 0 /* col offset 1 */, 20 /* row offset 1 */, 0 /* col offset 2 */, 12 /* row offset 2 */);
/* variables */
static int next_frame = 0;
static int skipped_frames = 0;
static unsigned long start_ms, curr_ms, next_frame_ms;
/* audio */
#include "esp32_audio_task.h"
/* MJPEG Video */
#include "mjpeg_decode_draw_task.h"
// pixel drawing callback
static int drawMCU(JPEGDRAW *pDraw)
{
// Serial.printf("Draw pos = (%d, %d), size = %d x %d\n", pDraw->x, pDraw->y, pDraw->iWidth, pDraw->iHeight);
unsigned long s = millis();
gfx->draw16bitRGBBitmap(pDraw->x, pDraw->y, pDraw->pPixels, pDraw->iWidth, pDraw->iHeight);
total_show_video_ms += millis() - s;
return 1;
} /* drawMCU() */
void setup()
{
disableCore0WDT();
WiFi.mode(WIFI_OFF);
Serial.begin(115200);
// while (!Serial);
// Init Display
gfx->begin(80000000);
gfx->fillScreen(0x0000);
#ifdef GFX_BL
pinMode(GFX_BL, OUTPUT);
digitalWrite(GFX_BL, HIGH);
#endif
Serial.println("Init I2S");
gfx->println("Init I2S");
#if defined(ESP32) && (CONFIG_IDF_TARGET_ESP32)
esp_err_t ret_val = i2s_init(I2S_NUM_0, 44100, -1 /* MCLK */, 25 /* SCLK */, 26 /* LRCK */, 32 /* DOUT */, -1 /* DIN */);
#elif defined(ESP32) && (CONFIG_IDF_TARGET_ESP32S2)
esp_err_t ret_val = i2s_init(I2S_NUM_0, 44100, -1 /* MCLK */, 4 /* SCLK */, 5 /* LRCK */, 18 /* DOUT */, -1 /* DIN */);
#elif defined(ESP32) && (CONFIG_IDF_TARGET_ESP32S3)
esp_err_t ret_val = i2s_init(I2S_NUM_0, 44100, 42 /* MCLK */, 46 /* SCLK */, 45 /* LRCK */, 43 /* DOUT */, 44 /* DIN */);
#elif defined(ESP32) && (CONFIG_IDF_TARGET_ESP32C3)
esp_err_t ret_val = i2s_init(I2S_NUM_0, 44100, -1 /* MCLK */, 10 /* SCLK */, 19 /* LRCK */, 18 /* DOUT */, -1 /* DIN */);
#endif
if (ret_val != ESP_OK)
{
Serial.printf("i2s_init failed: %d\n", ret_val);
}
i2s_zero_dma_buffer(I2S_NUM_0);
Serial.println("Init FS");
gfx->println("Init FS");
// if (!LittleFS.begin(false, "/root"))
// if (!SPIFFS.begin(false, "/root"))
// if (!FFat.begin(false, "/root"))
SPIClass spi = SPIClass(HSPI);
// spi.begin(14 /* SCK */, 2 /* MISO */, 15 /* MOSI */, 13 /* CS */);
spi.begin(14 /* SCK */, 4 /* MISO */, 15 /* MOSI */, 13 /* CS */);
if (!SD.begin(13, spi, 80000000))
// if ((!SD_MMC.begin("/root")) && (!SD_MMC.begin("/root")) && (!SD_MMC.begin("/root")) && (!SD_MMC.begin("/root"))) /* 4-bit SD bus mode */
// if ((!SD_MMC.begin("/root", true)) && (!SD_MMC.begin("/root", true)) && (!SD_MMC.begin("/root", true)) && (!SD_MMC.begin("/root", true))) /* 1-bit SD bus mode */
{
Serial.println("ERROR: File system mount failed!");
gfx->println("ERROR: File system mount failed!");
}
else
{
bool aac_file_available = false;
Serial.println("Open AAC file: " AAC_FILENAME);
gfx->println("Open AAC file: " AAC_FILENAME);
// File aFile = LittleFS.open(AAC_FILENAME);
// File aFile = SPIFFS.open(AAC_FILENAME);
// File aFile = FFat.open(AAC_FILENAME);
File aFile = SD.open(AAC_FILENAME);
// File aFile = SD_MMC.open(AAC_FILENAME);
if (aFile)
{
aac_file_available = true;
}
else
{
Serial.println("Open MP3 file: " MP3_FILENAME);
gfx->println("Open MP3 file: " MP3_FILENAME);
// aFile = LittleFS.open(MP3_FILENAME);
// aFile = SPIFFS.open(MP3_FILENAME);
// aFile = FFat.open(MP3_FILENAME);
aFile = SD.open(MP3_FILENAME);
// aFile = SD_MMC.open(MP3_FILENAME);
}
if (!aFile || aFile.isDirectory())
{
Serial.println("ERROR: Failed to open " AAC_FILENAME " or " MP3_FILENAME " file for reading");
gfx->println("ERROR: Failed to open " AAC_FILENAME " or " MP3_FILENAME " file for reading");
}
else
{
Serial.println("Open MJPEG file: " MJPEG_FILENAME);
gfx->println("Open MJPEG file: " MJPEG_FILENAME);
// File vFile = LittleFS.open(MJPEG_FILENAME);
// File vFile = SPIFFS.open(MJPEG_FILENAME);
// File vFile = FFat.open(MJPEG_FILENAME);
File vFile = SD.open(MJPEG_FILENAME);
// File vFile = SD_MMC.open(MJPEG_FILENAME);
if (!vFile || vFile.isDirectory())
{
Serial.println("ERROR: Failed to open " MJPEG_FILENAME " file for reading");
gfx->println("ERROR: Failed to open " MJPEG_FILENAME " file for reading");
}
else
{
Serial.println("Init video");
gfx->println("Init video");
mjpeg_setup(&vFile, MJPEG_BUFFER_SIZE, drawMCU,
false /* useBigEndian */, DECODEASSIGNCORE, DRAWASSIGNCORE);
Serial.println("Start play audio task");
gfx->println("Start play audio task");
BaseType_t ret_val;
if (aac_file_available)
{
ret_val = aac_player_task_start(&aFile, AUDIOASSIGNCORE);
}
else
{
ret_val = mp3_player_task_start(&aFile, AUDIOASSIGNCORE);
}
if (ret_val != pdPASS)
{
Serial.printf("Audio player task start failed: %d\n", ret_val);
gfx->printf("Audio player task start failed: %d\n", ret_val);
}
Serial.println("Start play video");
gfx->println("Start play video");
start_ms = millis();
curr_ms = millis();
next_frame_ms = start_ms + (++next_frame * 1000 / FPS / 2);
while (vFile.available() && mjpeg_read_frame()) // Read video
{
total_read_video_ms += millis() - curr_ms;
curr_ms = millis();
if (millis() < next_frame_ms) // check show frame or skip frame
{
// Play video
mjpeg_draw_frame();
total_decode_video_ms += millis() - curr_ms;
curr_ms = millis();
}
else
{
++skipped_frames;
Serial.println("Skip frame");
}
while (millis() < next_frame_ms)
{
vTaskDelay(pdMS_TO_TICKS(1));
}
curr_ms = millis();
next_frame_ms = start_ms + (++next_frame * 1000 / FPS);
}
int time_used = millis() - start_ms;
int total_frames = next_frame - 1;
Serial.println("AV end");
vFile.close();
aFile.close();
int played_frames = total_frames - skipped_frames;
float fps = 1000.0 * played_frames / time_used;
total_decode_audio_ms -= total_play_audio_ms;
// total_decode_video_ms -= total_show_video_ms;
Serial.printf("Played frames: %d\n", played_frames);
Serial.printf("Skipped frames: %d (%0.1f %%)\n", skipped_frames, 100.0 * skipped_frames / total_frames);
Serial.printf("Time used: %d ms\n", time_used);
Serial.printf("Expected FPS: %d\n", FPS);
Serial.printf("Actual FPS: %0.1f\n", fps);
Serial.printf("Read audio: %lu ms (%0.1f %%)\n", total_read_audio_ms, 100.0 * total_read_audio_ms / time_used);
Serial.printf("Decode audio: %lu ms (%0.1f %%)\n", total_decode_audio_ms, 100.0 * total_decode_audio_ms / time_used);
Serial.printf("Play audio: %lu ms (%0.1f %%)\n", total_play_audio_ms, 100.0 * total_play_audio_ms / time_used);
Serial.printf("Read video: %lu ms (%0.1f %%)\n", total_read_video_ms, 100.0 * total_read_video_ms / time_used);
Serial.printf("Decode video: %lu ms (%0.1f %%)\n", total_decode_video_ms, 100.0 * total_decode_video_ms / time_used);
Serial.printf("Show video: %lu ms (%0.1f %%)\n", total_show_video_ms, 100.0 * total_show_video_ms / time_used);
#define CHART_MARGIN 64
#define LEGEND_A_COLOR 0x1BB6
#define LEGEND_B_COLOR 0xFBE1
#define LEGEND_C_COLOR 0x2D05
#define LEGEND_D_COLOR 0xD125
#define LEGEND_E_COLOR 0x9337
#define LEGEND_F_COLOR 0x8AA9
#define LEGEND_G_COLOR 0xE3B8
#define LEGEND_H_COLOR 0x7BEF
#define LEGEND_I_COLOR 0xBDE4
#define LEGEND_J_COLOR 0x15F9
// gfx->setCursor(0, 0);
gfx->setTextColor(0xFFFF);
gfx->printf("Played frames: %d\n", played_frames);
gfx->printf("Skipped frames: %d (%0.1f %%)\n", skipped_frames, 100.0 * skipped_frames / total_frames);
gfx->printf("Time used: %d ms\n", time_used);
gfx->printf("Expected FPS: %d\n", FPS);
gfx->printf("Actual FPS: %0.1f\n\n", fps);
int16_t r1 = ((gfx->height() - CHART_MARGIN - CHART_MARGIN) / 2);
int16_t r2 = r1 / 2;
int16_t cx = gfx->width() - r1 - 10;
int16_t cy = r1 + CHART_MARGIN;
float arc_start1 = 0;
float arc_end1 = arc_start1 + max(2.0, 360.0 * total_read_audio_ms / time_used);
for (int i = arc_start1 + 1; i < arc_end1; i += 2)
{
gfx->fillArc(cx, cy, r1, r2, arc_start1 - 90.0, i - 90.0, LEGEND_A_COLOR);
}
gfx->fillArc(cx, cy, r1, r2, arc_start1 - 90.0, arc_end1 - 90.0, LEGEND_A_COLOR);
gfx->setTextColor(LEGEND_A_COLOR);
gfx->printf("Read audio: %lu ms (%0.1f %%)\n", total_read_audio_ms, 100.0 * total_read_audio_ms / time_used);
float arc_start2 = arc_end1;
float arc_end2 = arc_start2 + max(2.0, 360.0 * total_decode_audio_ms / time_used);
for (int i = arc_start2 + 1; i < arc_end2; i += 2)
{
gfx->fillArc(cx, cy, r1, r2, arc_start2 - 90.0, i - 90.0, LEGEND_B_COLOR);
}
gfx->fillArc(cx, cy, r1, r2, arc_start2 - 90.0, arc_end2 - 90.0, LEGEND_B_COLOR);
gfx->setTextColor(LEGEND_B_COLOR);
gfx->printf("Decode audio: %lu ms (%0.1f %%)\n", total_decode_audio_ms, 100.0 * total_decode_audio_ms / time_used);
gfx->setTextColor(LEGEND_J_COLOR);
gfx->printf("Play audio: %lu ms (%0.1f %%)\n", total_play_audio_ms, 100.0 * total_play_audio_ms / time_used);
float arc_start3 = arc_end2;
float arc_end3 = arc_start3 + max(2.0, 360.0 * total_read_video_ms / time_used);
for (int i = arc_start3 + 1; i < arc_end3; i += 2)
{
gfx->fillArc(cx, cy, r1, r2, arc_start3 - 90.0, i - 90.0, LEGEND_C_COLOR);
}
gfx->fillArc(cx, cy, r1, r2, arc_start3 - 90.0, arc_end3 - 90.0, LEGEND_C_COLOR);
gfx->setTextColor(LEGEND_C_COLOR);
gfx->printf("Read video: %lu ms (%0.1f %%)\n", total_read_video_ms, 100.0 * total_read_video_ms / time_used);
float arc_start4 = arc_end3;
float arc_end4 = arc_start4 + max(2.0, 360.0 * total_show_video_ms / time_used);
for (int i = arc_start4 + 1; i < arc_end4; i += 2)
{
gfx->fillArc(cx, cy, r1, r2, arc_start4 - 90.0, i - 90.0, LEGEND_D_COLOR);
}
gfx->fillArc(cx, cy, r1, r2, arc_start4 - 90.0, arc_end4 - 90.0, LEGEND_D_COLOR);
gfx->setTextColor(LEGEND_D_COLOR);
gfx->printf("Show video: %lu ms (%0.1f %%)\n", total_show_video_ms, 100.0 * total_show_video_ms / time_used);
float arc_start5 = 0;
float arc_end5 = arc_start5 + max(2.0, 360.0 * total_decode_video_ms / time_used);
for (int i = arc_start5 + 1; i < arc_end5; i += 2)
{
gfx->fillArc(cx, cy, r2, 0, arc_start5 - 90.0, i - 90.0, LEGEND_E_COLOR);
}
gfx->fillArc(cx, cy, r2, 0, arc_start5 - 90.0, arc_end5 - 90.0, LEGEND_E_COLOR);
gfx->setTextColor(LEGEND_E_COLOR);
gfx->printf("Decode video: %lu ms (%0.1f %%)\n", total_decode_video_ms, 100.0 * total_decode_video_ms / time_used);
}
// delay(60000);
#ifdef GFX_BL
// digitalWrite(GFX_BL, LOW);
#endif
// gfx->displayOff();
// esp_deep_sleep_start();
}
}
}
void loop()
{
}
esp32_audio_task.h
#include "driver/i2s.h"
#include "AACDecoderHelix.h"
#include "MP3DecoderHelix.h"
static unsigned long total_read_audio_ms = 0;
static unsigned long total_decode_audio_ms = 0;
static unsigned long total_play_audio_ms = 0;
static i2s_port_t _i2s_num;
static esp_err_t i2s_init(i2s_port_t i2s_num, uint32_t sample_rate,
int mck_io_num, /*!< MCK in out pin. Note that ESP32 supports setting MCK on GPIO0/GPIO1/GPIO3 only*/
int bck_io_num, /*!< BCK in out pin*/
int ws_io_num, /*!< WS in out pin*/
int data_out_num, /*!< DATA out pin*/
int data_in_num /*!< DATA in pin*/
)
{
_i2s_num = i2s_num;
esp_err_t ret_val = ESP_OK;
i2s_config_t i2s_config;
i2s_config.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX);
i2s_config.sample_rate = sample_rate;
i2s_config.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT;
i2s_config.channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT;
i2s_config.communication_format = I2S_COMM_FORMAT_STAND_I2S;
i2s_config.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1;
i2s_config.dma_buf_count = 8;
i2s_config.dma_buf_len = 160;
i2s_config.use_apll = false;
i2s_config.tx_desc_auto_clear = true;
i2s_config.fixed_mclk = 0;
i2s_config.mclk_multiple = I2S_MCLK_MULTIPLE_768;
i2s_config.bits_per_chan = I2S_BITS_PER_CHAN_16BIT;
i2s_pin_config_t pin_config;
pin_config.mck_io_num = mck_io_num;
pin_config.bck_io_num = bck_io_num;
pin_config.ws_io_num = ws_io_num;
pin_config.data_out_num = data_out_num;
pin_config.data_in_num = data_in_num;
ret_val |= i2s_driver_install(i2s_num, &i2s_config, 0, NULL);
ret_val |= i2s_set_pin(i2s_num, &pin_config);
return ret_val;
}
static int _samprate = 0;
static void aacAudioDataCallback(AACFrameInfo &info, int16_t *pwm_buffer, size_t len)
{
unsigned long s = millis();
if (_samprate != info.sampRateOut)
{
// log_i("bitRate: %d, nChans: %d, sampRateCore: %d, sampRateOut: %d, bitsPerSample: %d, outputSamps: %d, profile: %d, tnsUsed: %d, pnsUsed: %d",
// info.bitRate, info.nChans, info.sampRateCore, info.sampRateOut, info.bitsPerSample, info.outputSamps, info.profile, info.tnsUsed, info.pnsUsed);
i2s_set_clk(_i2s_num, info.sampRateOut /* sample_rate */, info.bitsPerSample /* bits_cfg */, (info.nChans == 2) ? I2S_CHANNEL_STEREO : I2S_CHANNEL_MONO /* channel */);
_samprate = info.sampRateOut;
}
size_t i2s_bytes_written = 0;
i2s_write(_i2s_num, pwm_buffer, len * 2, &i2s_bytes_written, portMAX_DELAY);
// log_i("len: %d, i2s_bytes_written: %d", len, i2s_bytes_written);
total_play_audio_ms += millis() - s;
}
static void mp3AudioDataCallback(MP3FrameInfo &info, int16_t *pwm_buffer, size_t len)
{
unsigned long s = millis();
if (_samprate != info.samprate)
{
log_i("bitrate: %d, nChans: %d, samprate: %d, bitsPerSample: %d, outputSamps: %d, layer: %d, version: %d",
info.bitrate, info.nChans, info.samprate, info.bitsPerSample, info.outputSamps, info.layer, info.version);
i2s_set_clk(_i2s_num, info.samprate /* sample_rate */, info.bitsPerSample /* bits_cfg */, (info.nChans == 2) ? I2S_CHANNEL_STEREO : I2S_CHANNEL_MONO /* channel */);
_samprate = info.samprate;
}
size_t i2s_bytes_written = 0;
i2s_write(_i2s_num, pwm_buffer, len * 2, &i2s_bytes_written, portMAX_DELAY);
// log_i("len: %d, i2s_bytes_written: %d", len, i2s_bytes_written);
total_play_audio_ms += millis() - s;
}
static uint8_t _frame[MP3_MAX_FRAME_SIZE]; // MP3_MAX_FRAME_SIZE is smaller, so always use MP3_MAX_FRAME_SIZE
static libhelix::AACDecoderHelix _aac(aacAudioDataCallback);
static void aac_player_task(void *pvParam)
{
Stream *input = (Stream *)pvParam;
int r, w;
unsigned long ms = millis();
while (r = input->readBytes(_frame, MP3_MAX_FRAME_SIZE))
{
total_read_audio_ms += millis() - ms;
ms = millis();
while (r > 0)
{
w = _aac.write(_frame, r);
// log_i("r: %d, w: %d\n", r, w);
r -= w;
}
total_decode_audio_ms += millis() - ms;
ms = millis();
}
log_i("AAC stop.");
vTaskDelete(NULL);
}
static libhelix::MP3DecoderHelix _mp3(mp3AudioDataCallback);
static void mp3_player_task(void *pvParam)
{
Stream *input = (Stream *)pvParam;
int r, w;
unsigned long ms = millis();
while (r = input->readBytes(_frame, MP3_MAX_FRAME_SIZE))
{
total_read_audio_ms += millis() - ms;
ms = millis();
while (r > 0)
{
w = _mp3.write(_frame, r);
// log_i("r: %d, w: %d\n", r, w);
r -= w;
}
total_decode_audio_ms += millis() - ms;
ms = millis();
}
log_i("MP3 stop.");
vTaskDelete(NULL);
}
static BaseType_t aac_player_task_start(Stream *input, BaseType_t audioAssignCore)
{
_aac.begin();
return xTaskCreatePinnedToCore(
(TaskFunction_t)aac_player_task,
(const char *const)"AAC Player Task",
(const uint32_t)2000,
(void *const)input,
(UBaseType_t)configMAX_PRIORITIES - 1,
(TaskHandle_t *const)NULL,
(const BaseType_t)audioAssignCore);
}
static BaseType_t mp3_player_task_start(Stream *input, BaseType_t audioAssignCore)
{
_mp3.begin();
return xTaskCreatePinnedToCore(
(TaskFunction_t)mp3_player_task,
(const char *const)"MP3 Player Task",
(const uint32_t)2000,
(void *const)input,
(UBaseType_t)configMAX_PRIORITIES - 1,
(TaskHandle_t *const)NULL,
(const BaseType_t)audioAssignCore);
}
mjpeg_decode_draw_task.h
#define READ_BUFFER_SIZE 1024
// #define MAXOUTPUTSIZE (MAX_BUFFERED_PIXELS / 16 / 16)
#define MAXOUTPUTSIZE (288 / 3 / 16)
#define NUMBER_OF_DECODE_BUFFER 3
#define NUMBER_OF_DRAW_BUFFER 9
#include <FS.h>
#include <JPEGDEC.h>
typedef struct
{
int32_t size;
uint8_t *buf;
} mjpegBuf;
typedef struct
{
xQueueHandle xqh;
JPEG_DRAW_CALLBACK *drawFunc;
} paramDrawTask;
typedef struct
{
xQueueHandle xqh;
mjpegBuf *mBuf;
JPEG_DRAW_CALLBACK *drawFunc;
} paramDecodeTask;
static JPEGDRAW jpegdraws[NUMBER_OF_DRAW_BUFFER];
static int _draw_queue_cnt = 0;
static JPEGDEC _jpegDec;
static xQueueHandle _xqh;
static bool _useBigEndian;
static unsigned long total_read_video_ms = 0;
static unsigned long total_decode_video_ms = 0;
static unsigned long total_show_video_ms = 0;
Stream *_input;
int32_t _mjpegBufSize;
uint8_t *_read_buf;
int32_t _mjpeg_buf_offset = 0;
TaskHandle_t _decodeTask;
TaskHandle_t _draw_task;
paramDecodeTask _pDecodeTask;
paramDrawTask _pDrawTask;
uint8_t *_mjpeg_buf;
uint8_t _mBufIdx = 0;
int32_t _inputindex = 0;
int32_t _buf_read;
int32_t _remain = 0;
mjpegBuf _mjpegBufs[NUMBER_OF_DECODE_BUFFER];
static int queueDrawMCU(JPEGDRAW *pDraw)
{
int len = pDraw->iWidth * pDraw->iHeight * 2;
JPEGDRAW *j = &jpegdraws[_draw_queue_cnt % NUMBER_OF_DRAW_BUFFER];
j->x = pDraw->x;
j->y = pDraw->y;
j->iWidth = pDraw->iWidth;
j->iHeight = pDraw->iHeight;
memcpy(j->pPixels, pDraw->pPixels, len);
// log_i("queueDrawMCU start.");
++_draw_queue_cnt;
xQueueSend(_xqh, &j, portMAX_DELAY);
// log_i("queueDrawMCU end.");
return 1;
}
static void decode_task(void *arg)
{
paramDecodeTask *p = (paramDecodeTask *)arg;
mjpegBuf *mBuf;
log_i("decode_task start.");
while (xQueueReceive(p->xqh, &mBuf, portMAX_DELAY))
{
// log_i("mBuf->size: %d", mBuf->size);
// log_i("mBuf->buf start: %X %X, end: %X, %X.", mBuf->buf[0], mBuf->buf[1], mBuf->buf[mBuf->size - 2], mBuf->buf[mBuf->size - 1]);
unsigned long s = millis();
_jpegDec.openRAM(mBuf->buf, mBuf->size, p->drawFunc);
// _jpegDec.setMaxOutputSize(MAXOUTPUTSIZE);
if (_useBigEndian)
{
_jpegDec.setPixelType(RGB565_BIG_ENDIAN);
}
_jpegDec.setMaxOutputSize(MAXOUTPUTSIZE);
_jpegDec.decode(0, 0, 0);
_jpegDec.close();
total_decode_video_ms += millis() - s;
}
vQueueDelete(p->xqh);
log_i("decode_task end.");
vTaskDelete(NULL);
}
static void draw_task(void *arg)
{
paramDrawTask *p = (paramDrawTask *)arg;
JPEGDRAW *pDraw;
log_i("draw_task start.");
while (xQueueReceive(p->xqh, &pDraw, portMAX_DELAY))
{
// log_i("draw_task work start: x: %d, y: %d, iWidth: %d, iHeight: %d.", pDraw->x, pDraw->y, pDraw->iWidth, pDraw->iHeight);
p->drawFunc(pDraw);
// log_i("draw_task work end.");
}
vQueueDelete(p->xqh);
log_i("draw_task end.");
vTaskDelete(NULL);
}
bool mjpeg_setup(Stream *input, int32_t mjpegBufSize, JPEG_DRAW_CALLBACK *pfnDraw,
bool useBigEndian, BaseType_t decodeAssignCore, BaseType_t drawAssignCore)
{
_input = input;
_mjpegBufSize = mjpegBufSize;
_useBigEndian = useBigEndian;
for (int i = 0; i < NUMBER_OF_DECODE_BUFFER; ++i)
{
_mjpegBufs[i].buf = (uint8_t *)malloc(mjpegBufSize);
if (_mjpegBufs[i].buf)
{
log_i("#%d decode buffer allocated.", i);
}
else
{
log_e("#%d decode buffer allocat failed.", i);
}
}
_mjpeg_buf = _mjpegBufs[_mBufIdx].buf;
if (!_read_buf)
{
_read_buf = (uint8_t *)malloc(READ_BUFFER_SIZE);
}
if (_read_buf)
{
log_i("Read buffer allocated.");
}
_xqh = xQueueCreate(NUMBER_OF_DRAW_BUFFER, sizeof(JPEGDRAW));
_pDrawTask.xqh = _xqh;
_pDrawTask.drawFunc = pfnDraw;
_pDecodeTask.xqh = xQueueCreate(NUMBER_OF_DECODE_BUFFER, sizeof(mjpegBuf));
_pDecodeTask.drawFunc = queueDrawMCU;
xTaskCreatePinnedToCore(
(TaskFunction_t)decode_task,
(const char *const)"MJPEG decode Task",
(const uint32_t)2000,
(void *const)&_pDecodeTask,
(UBaseType_t)configMAX_PRIORITIES - 1,
(TaskHandle_t *const)&_decodeTask,
(const BaseType_t)decodeAssignCore);
xTaskCreatePinnedToCore(
(TaskFunction_t)draw_task,
(const char *const)"MJPEG Draw Task",
(const uint32_t)2000,
(void *const)&_pDrawTask,
(UBaseType_t)configMAX_PRIORITIES - 1,
(TaskHandle_t *const)&_draw_task,
(const BaseType_t)drawAssignCore);
for (int i = 0; i < NUMBER_OF_DRAW_BUFFER; i++)
{
if (!jpegdraws[i].pPixels)
{
jpegdraws[i].pPixels = (uint16_t *)heap_caps_malloc(MAXOUTPUTSIZE * 16 * 16 * 2, MALLOC_CAP_DMA);
}
if (jpegdraws[i].pPixels)
{
log_i("#%d draw buffer allocated.", i);
}
else
{
log_e("#%d draw buffer allocat failed.", i);
}
}
return true;
}
bool mjpeg_read_frame()
{
if (_inputindex == 0)
{
_buf_read = _input->readBytes(_read_buf, READ_BUFFER_SIZE);
_inputindex += _buf_read;
}
_mjpeg_buf_offset = 0;
int i = 0;
bool found_FFD8 = false;
while ((_buf_read > 0) && (!found_FFD8))
{
i = 0;
while ((i < _buf_read) && (!found_FFD8))
{
if ((_read_buf[i] == 0xFF) && (_read_buf[i + 1] == 0xD8)) // JPEG header
{
// log_i("Found FFD8 at: %d.", i);
found_FFD8 = true;
}
++i;
}
if (found_FFD8)
{
--i;
}
else
{
_buf_read = _input->readBytes(_read_buf, READ_BUFFER_SIZE);
}
}
uint8_t *_p = _read_buf + i;
_buf_read -= i;
bool found_FFD9 = false;
if (_buf_read > 0)
{
i = 3;
while ((_buf_read > 0) && (!found_FFD9))
{
if ((_mjpeg_buf_offset > 0) && (_mjpeg_buf[_mjpeg_buf_offset - 1] == 0xFF) && (_p[0] == 0xD9)) // JPEG trailer
{
found_FFD9 = true;
}
else
{
while ((i < _buf_read) && (!found_FFD9))
{
if ((_p[i] == 0xFF) && (_p[i + 1] == 0xD9)) // JPEG trailer
{
found_FFD9 = true;
++i;
}
++i;
}
}
// log_i("i: %d", i);
memcpy(_mjpeg_buf + _mjpeg_buf_offset, _p, i);
_mjpeg_buf_offset += i;
int32_t o = _buf_read - i;
if (o > 0)
{
// log_i("o: %d", o);
memcpy(_read_buf, _p + i, o);
_buf_read = _input->readBytes(_read_buf + o, READ_BUFFER_SIZE - o);
_p = _read_buf;
_inputindex += _buf_read;
_buf_read += o;
// log_i("_buf_read: %d", _buf_read);
}
else
{
_buf_read = _input->readBytes(_read_buf, READ_BUFFER_SIZE);
_p = _read_buf;
_inputindex += _buf_read;
}
i = 0;
}
if (found_FFD9)
{
// log_i("Found FFD9 at: %d.", _mjpeg_buf_offset);
if (_mjpeg_buf_offset > _mjpegBufSize) {
log_e("_mjpeg_buf_offset(%d) > _mjpegBufSize (%d)", _mjpeg_buf_offset, _mjpegBufSize);
}
return true;
}
}
return false;
}
bool mjpeg_draw_frame()
{
mjpegBuf *mBuf = &_mjpegBufs[_mBufIdx];
mBuf->size = _mjpeg_buf_offset;
// log_i("_mjpegBufs[%d].size: %d.", _mBufIdx, _mjpegBufs[_mBufIdx].size);
// log_i("_mjpegBufs[%d].buf start: %X %X, end: %X, %X.", _mjpegBufs, _mjpegBufs[_mBufId].buf[0], _mjpegBufs[_mBufIdx].buf[1], _mjpegBufs[_mBufIdx].buf[_mjpeg_buf_offset - 2], _mjpegBufs[_mBufIdx].buf[_mjpeg_buf_offset - 1]);
xQueueSend(_pDecodeTask.xqh, &mBuf, portMAX_DELAY);
++_mBufIdx;
if (_mBufIdx >= NUMBER_OF_DECODE_BUFFER)
{
_mBufIdx = 0;
}
_mjpeg_buf = _mjpegBufs[_mBufIdx].buf;
// log_i("queue decode_task end");
return true;
}