Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
327 changes: 150 additions & 177 deletions SerialPrograms/CMakeLists.txt

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions SerialPrograms/Source/CommonFramework/Globals.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ enum class FeedbackType{
OPTIONAL_, // Naming conflict with macro.
REQUIRED,
VIDEO_AUDIO,
VIDEO_AUDIO_GBA,
};


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ CollapsibleGroupBox* make_panel_header(
);
break;
case FeedbackType::VIDEO_AUDIO:
case FeedbackType::VIDEO_AUDIO_GBA:
text = new QLabel(
"<font color=\"green\">(This program requires video and audio feedback. Please make sure you choose the correct capture device, as well as the correct audio device.)</font>",
header
Expand Down
60 changes: 60 additions & 0 deletions SerialPrograms/Source/CommonFramework/Tools/BlackBorderCheck.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/* Black Border Check
*
* From: https://github.com/PokemonAutomation/Arduino-Source
*
*/

#include "Common/Cpp/Exceptions.h"
#include "CommonFramework/ImageTypes/ImageRGB32.h"
#include "CommonFramework/VideoPipeline/VideoFeed.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "CommonFramework/Tools/ConsoleHandle.h"
#include "CommonFramework/Inference/BlackBorderDetector.h"
#include "CommonFramework/Inference/BlackBorderGBADetector.h"
#include "BlackBorderCheck.h"

namespace PokemonAutomation{


void start_program_video_check(ConsoleHandle& console, FeedbackType feedback){
if (feedback == FeedbackType::NONE){
return;
}

VideoSnapshot screen = console.video().snapshot();

if (!screen){
if (feedback == FeedbackType::REQUIRED || feedback == FeedbackType::VIDEO_AUDIO || feedback == FeedbackType::VIDEO_AUDIO_GBA){
throw UserSetupError(console, "This program requires video feedback. Please make sure the video is working.");
}
return;
}

if (feedback != FeedbackType::VIDEO_AUDIO_GBA) { //GB, GBC in fullscreen will reach the top and bottom of the screen
BlackBorderDetector detector;
VideoOverlaySet set(console);
detector.make_overlays(set);

if (detector.detect(screen)) {
throw UserSetupError(console, "Black border detected! Please set your screen size to 100% in the TV Settings on your Nintendo Switch.");
}
}
else {
BlackBorderGBADetector detector;
VideoOverlaySet set(console);
detector.make_overlays(set);

if (detector.detect(screen)) {
throw UserSetupError(console, "Black border detected! Please set your screen size to 100% in the TV Settings on your Nintendo Switch.");
}
}
}
void start_program_video_check(FixedLimitVector<ConsoleHandle>& consoles, FeedbackType feedback){
for (ConsoleHandle& console : consoles){
start_program_video_check(console, feedback);
}
}



}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/* Black Border Detector
*
* From: https://github.com/PokemonAutomation/Arduino-Source
*
*/

#include "CommonFramework/ImageTools/ImageStats.h"
#include "CommonFramework/ImageTools/SolidColorTest.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "BlackBorderGBADetector.h"

#include <iostream>
using std::cout;
using std::endl;

namespace PokemonAutomation{


BlackBorderGBADetector::BlackBorderGBADetector()
: m_top(0.126, 0.055, 0.748, 0.006)
, m_bottom(0.124, 0.940, 0.751, 0.004)
, m_left(0.126, 0.055, 0.002, 0.888)
, m_right(0.871, 0.055, 0.003, 0.888)
// , m_body(0.100, 0.100, 0.800, 0.800)
{}

void BlackBorderGBADetector::make_overlays(VideoOverlaySet& items) const{
items.add(COLOR_RED, m_top);
items.add(COLOR_RED, m_bottom);
items.add(COLOR_RED, m_left);
items.add(COLOR_RED, m_right);
// items.add(COLOR_RED, m_body);
}
bool BlackBorderGBADetector::detect(const ImageViewRGB32& screen) const{
const double MAX_SUM = 50;
const double MAX_STDDEV = 20;

ImageStats top = image_stats(extract_box_reference(screen, m_top));
// cout << "top = " << top.average << top.stddev << endl;
// extract_box(screen, m_top).save("top.png");
if (!is_black(top, MAX_SUM, MAX_STDDEV)){
return false;
}
ImageStats bottom = image_stats(extract_box_reference(screen, m_bottom));
// cout << "bottom = " << bottom.average << bottom.stddev << endl;
if (!is_black(bottom, MAX_SUM, MAX_STDDEV)){
return false;
}
ImageStats left = image_stats(extract_box_reference(screen, m_left));
// cout << "left = " << left.average << left.stddev << endl;
if (!is_black(left, MAX_SUM, MAX_STDDEV)){
return false;
}
ImageStats right = image_stats(extract_box_reference(screen, m_right));
// cout << "right = " << right.average << right.stddev << endl;
if (!is_black(right, MAX_SUM, MAX_STDDEV)){
return false;
}
// ImageStats body = image_stats(extract_box_reference(screen, m_body));
// cout << "body = " << body.average << body.stddev << endl;
// if (is_black(right, 30, 30)){
// return false;
// }


// for (int c = 0; c < screen.width(); c++){
// QRgb pixel = screen.pixel(c, 0);
// cout << "(" << qRed(pixel) << "," << qGreen(pixel) << "," << qBlue(pixel) << ")";
// }

return true;
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/* Black Screen Detector
*
* From: https://github.com/PokemonAutomation/Arduino-Source
*
*/

#ifndef PokemonAutomation_CommonFramework_BlackBorderGBADetector_H
#define PokemonAutomation_CommonFramework_BlackBorderGBADetector_H

#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/Inference/VisualDetector.h"

namespace PokemonAutomation{


class BlackBorderGBADetector : public StaticScreenDetector{
public:
BlackBorderGBADetector();

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) const override;

private:
ImageFloatBox m_top;
ImageFloatBox m_bottom;
ImageFloatBox m_left;
ImageFloatBox m_right;
// ImageFloatBox m_body;
};



}
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -15,43 +15,50 @@ namespace NintendoSwitch{

// constexpr Color COLOR_GREEN2(0xff00aa00);

Color pick_color(FeedbackType feedback){
switch (feedback){
case FeedbackType::NONE:
return COLOR_BLUE;
case FeedbackType::OPTIONAL_:
return COLOR_PURPLE;
case FeedbackType::REQUIRED:
return COLOR_DARKGREEN;
case FeedbackType::VIDEO_AUDIO:
return COLOR_RED;
Color pick_color(FeedbackType feedback, PABotBaseLevel size){
switch (size){
case PABotBaseLevel::NOT_PABOTBASE:
return Color();
case PABotBaseLevel::PABOTBASE_12KB:
if (feedback == FeedbackType::REQUIRED){
return COLOR_DARKGREEN;
}else if (feedback == FeedbackType::VIDEO_AUDIO || feedback == FeedbackType::VIDEO_AUDIO_GBA){
return COLOR_GREEN2;
}else{
return COLOR_BLUE;
}
case PABotBaseLevel::PABOTBASE_31KB:
return (feedback == FeedbackType::REQUIRED || feedback == FeedbackType::VIDEO_AUDIO || feedback == FeedbackType::VIDEO_AUDIO_GBA) ? COLOR_PURPLE : COLOR_RED;
}
return Color();
}



const std::string SwitchSystemOption::JSON_CONTROLLER = "Controller";
const std::string SwitchSystemOption::JSON_SERIAL = "Serial";
const std::string SwitchSystemOption::JSON_CAMERA = "Camera";
const std::string SwitchSystemOption::JSON_AUDIO = "Audio";
const std::string SwitchSystemOption::JSON_OVERLAY = "Overlay";


SwitchSystemOption::SwitchSystemOption(
const ControllerRequirements& requirements,
PABotBaseLevel min_pabotbase,
bool allow_commands_while_running
)
: m_requirements(requirements)
, m_allow_commands_while_running(allow_commands_while_running)
: m_allow_commands_while_running(allow_commands_while_running)
, m_serial(min_pabotbase)
, m_camera(DEFAULT_RESOLUTION)
// , m_audio()
{}
SwitchSystemOption::SwitchSystemOption(
const ControllerRequirements& requirements,
PABotBaseLevel min_pabotbase,
bool allow_commands_while_running,
const JsonValue& json
)
: SwitchSystemOption(requirements, allow_commands_while_running)
: SwitchSystemOption(
min_pabotbase,
allow_commands_while_running
)
{
load_json(json);
}
Expand All @@ -62,9 +69,9 @@ void SwitchSystemOption::load_json(const JsonValue& json){
}
// json_get_bool(m_settings_visible, obj, "SettingsVisible");
const JsonValue* value;
value = obj->get_value(JSON_CONTROLLER);
value = obj->get_value(JSON_SERIAL);
if (value){
m_controller.load_json(*value);
m_serial.load_json(*value);
}
value = obj->get_value(JSON_CAMERA);
if (value){
Expand All @@ -82,7 +89,7 @@ void SwitchSystemOption::load_json(const JsonValue& json){
JsonValue SwitchSystemOption::to_json() const{
JsonObject root;
// root.insert("SettingsVisible", m_settings_visible);
root[JSON_CONTROLLER] = m_controller.to_json();
root[JSON_SERIAL] = m_serial.to_json();
root[JSON_CAMERA] = m_camera.to_json();
root[JSON_AUDIO] = m_audio.to_json();
root[JSON_OVERLAY] = m_overlay.to_json();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
*
*/

#include "CommonTools/Images/SolidColorTest.h"
#include "CommonTools/Images/ImageFilter.h"
#include "CommonFramework/ImageTools/SolidColorTest.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/ImageTools/ImageFilter.h"
#include "CommonFramework/ImageTypes/ImageRGB32.h"
#include "CommonFramework/ImageTools/ImageStats.h"
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
Expand Down Expand Up @@ -76,17 +76,13 @@ bool BattleMenuDetector::detect(const ImageViewRGB32& screen) const{
}


AdvanceBattleDialogDetector::AdvanceBattleDialogDetector(Color color)
AdvanceDialogDetector::AdvanceDialogDetector(Color color)
: m_dialog_box(0.156, 0.715, 0.686, 0.193)
, m_left_box(0.156, 0.724, 0.010, 0.176)
, m_right_box(0.836, 0.717, 0.008, 0.189)
{}
void AdvanceBattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
void AdvanceDialogDetector::make_overlays(VideoOverlaySet& items) const{
items.add(COLOR_RED, m_dialog_box);
items.add(COLOR_RED, m_left_box);
items.add(COLOR_RED, m_right_box);
}
bool AdvanceBattleDialogDetector::detect(const ImageViewRGB32& screen) const{
bool AdvanceDialogDetector::detect(const ImageViewRGB32& screen) const{
const bool replace_color_within_range = false;

//Filter out background
Expand All @@ -103,15 +99,7 @@ bool AdvanceBattleDialogDetector::detect(const ImageViewRGB32& screen) const{
cout << stats.average.b << endl;
*/

ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);

if (is_solid(left_image, { 0.335, 0.331, 0.332 })
&& is_solid(right_image, { 0.335, 0.331, 0.332 })
&& (stats.average.r > stats.average.b + 200)
&& (stats.average.r > stats.average.g + 200)
)
{
if ((stats.average.r > stats.average.b + 200) && (stats.average.r > stats.average.g + 200)){
return true;
}
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@

#include <chrono>
#include <atomic>
//#include "CommonFramework/Logging/Logger.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "Common/Cpp/Color.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonTools/VisualDetector.h"
#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
#include "CommonFramework/InferenceInfra/VisualInferenceCallback.h"
#include "CommonFramework/Inference/VisualDetector.h"

namespace PokemonAutomation{
class CancellableScope;
Expand All @@ -22,7 +23,7 @@ namespace NintendoSwitch{
namespace PokemonRSE{

/*
// TODO: Detect that a dialog box is on screen by looking for the white of the box
// Detect that a dialog box is on screen by looking for the white of the box
class DialogDetector : public StaticScreenDetector{
public:
DialogDetector(Color color);
Expand Down Expand Up @@ -63,7 +64,7 @@ class BattleDialogWatcher : public DetectorToFinder<BattleDialogDetector>{


// Battle menu is up when it is white on the right and teal on the left
// For emerald the text is more flush to the left, so we target the right of the battle menu box
// For emerald the test is more flush to the left, so we target the right of the battle menu box
class BattleMenuDetector : public StaticScreenDetector{
public:
BattleMenuDetector(Color color);
Expand All @@ -87,27 +88,24 @@ class BattleMenuWatcher : public DetectorToFinder<BattleMenuDetector>{
// Detect the red advancement arrow by filtering for red.
// This works for now, I don't think there's colored text?
// TODO: Change this to detect that the dialog arrow is in the dialog box by filtering for the red arrow
class AdvanceBattleDialogDetector : public StaticScreenDetector{
class AdvanceDialogDetector : public StaticScreenDetector{
public:
AdvanceBattleDialogDetector(Color color);
AdvanceDialogDetector(Color color);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) const override;

private:
ImageFloatBox m_dialog_box;
ImageFloatBox m_left_box;
ImageFloatBox m_right_box;
};
class AdvanceBattleDialogWatcher : public DetectorToFinder<AdvanceBattleDialogDetector>{
class AdvanceDialogWatcher : public DetectorToFinder<AdvanceDialogDetector>{
public:
AdvanceBattleDialogWatcher(Color color)
: DetectorToFinder("AdvanceBattleDialogWatcher", std::chrono::milliseconds(250), color)
AdvanceDialogWatcher(Color color)
: DetectorToFinder("AdvanceDialogWatcher", std::chrono::milliseconds(250), color)
{}
};

// Future note: when given a choice popup, there is no advance arrow.
// FRLG doesn't have an advance arrow, that's a future issue.
// when given a choice popup, there is no advance arrow


}
Expand Down
Loading
Loading