From e39d11c4f27ca63ca22c8a1da6378b831da3b18b Mon Sep 17 00:00:00 2001 From: vutukuri15 Date: Fri, 21 Feb 2025 13:35:44 -0800 Subject: [PATCH] Implemented Automatic Detecting Chip Colors and Counts --- .gitignore | 2 + app.json | 3 +- app/index.tsx | 18 +- babel.config.js | 14 + components/ChipDetection.tsx | 149 ++++-- components/ChipDistributionSummary.tsx | 20 +- package-lock.json | 670 ++++++++++++++++++++++++- package.json | 9 +- 8 files changed, 837 insertions(+), 48 deletions(-) create mode 100644 babel.config.js diff --git a/.gitignore b/.gitignore index c9d575d..3387e8f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ yarn-error.* *.tsbuildinfo app-example +android +.env \ No newline at end of file diff --git a/app.json b/app.json index a619d4b..c270037 100644 --- a/app.json +++ b/app.json @@ -15,7 +15,8 @@ "adaptiveIcon": { "foregroundImage": "./assets/images/adaptive-icon.png", "backgroundColor": "#ffffff" - } + }, + "package": "com.anonymous.pokerchipshelper" }, "web": { "bundler": "metro", diff --git a/app/index.tsx b/app/index.tsx index 77fdd18..f1bfc4a 100644 --- a/app/index.tsx +++ b/app/index.tsx @@ -5,11 +5,13 @@ import BuyInSelector from "@/components/BuyInSelector"; import ChipsSelector from "@/components/ChipsSelector"; import ChipDistributionSummary from "@/components/ChipDistributionSummary"; import ChipDetection from "@/components/ChipDetection"; + const IndexScreen = () => { const [playerCount, setPlayerCount] = useState(2); const [buyInAmount, setBuyInAmount] = useState(null); const [numberOfChips, setNumberOfChips] = useState(5); const [totalChipsCount, setTotalChipsCount] = useState([]); + const handleSave = () => { if (buyInAmount === null) { Alert.alert("Error", "Please select a valid buy-in amount"); @@ -20,6 +22,16 @@ const IndexScreen = () => { ); } }; + + // Update chip count based on detection or manual edit + const updateChipCount = (chipData: { [color: string]: number }) => { + // Convert the chip data from the API response or manual edit to a count array + const chipCountArray = Object.entries(chipData).map( + ([color, count]) => count + ); + setTotalChipsCount(chipCountArray); // Update the parent component's state + }; + return ( @@ -30,10 +42,7 @@ const IndexScreen = () => { setPlayerCount={setPlayerCount} /> - + { ); }; + export default IndexScreen; diff --git a/babel.config.js b/babel.config.js new file mode 100644 index 0000000..44bfac7 --- /dev/null +++ b/babel.config.js @@ -0,0 +1,14 @@ +module.exports = { + presets: ["module:metro-react-native-babel-preset"], + plugins: [ + [ + "module:react-native-dotenv", + { + moduleName: "@env", + path: ".env", + safe: true, + allowUndefined: false, + }, + ], + ], +}; diff --git a/components/ChipDetection.tsx b/components/ChipDetection.tsx index 4c02ab0..dac07c9 100644 --- a/components/ChipDetection.tsx +++ b/components/ChipDetection.tsx @@ -1,46 +1,137 @@ import React, { useState } from "react"; -import { View, Text } from "react-native"; -import { MaterialIcons } from "@expo/vector-icons"; +import { + View, + Button, + Image, + ActivityIndicator, + Text, + ScrollView, +} from "react-native"; import * as ImagePicker from "expo-image-picker"; +import { API_KEY } from "@env"; -/** - The best way forward for this component is likely to send the image chosen to an AI + NLP API. - Google cloud vision is likely a good choice, as I think it offers some sort of free tier or trial usage for free, as long as it can also support NLP prompts - We need to thoughtfully prompt the API and ask it to return data in a well formatted JSON, or return an error if the image supplied is unable to be read, or otherwise out of context - We could also ask it to return a "confidence" level as a percentage, if the user may find that helpful +const ChipDetection = ({ updateChipCount }) => { + const [imageUri, setImageUri] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [lastDetectedChips, setLastDetectedChips] = useState({}); - */ + // Ensure early return does not break hooks + const requestCameraPermissions = async () => { + const cameraPermission = await ImagePicker.requestCameraPermissionsAsync(); + return cameraPermission.granted; + }; -const ChipDetection = ({ - totalChipsCount, - setTotalChipsCount, -}: { - totalChipsCount: number[]; - setTotalChipsCount: React.Dispatch>; -}) => { - const [image, setImage] = useState(null); const pickImage = async () => { - const { status } = await ImagePicker.requestMediaLibraryPermissionsAsync(); - if (status !== "granted") { - alert("Permission denied!"); - return; - } - - const result = await ImagePicker.launchCameraAsync({ - allowsEditing: true, + const result = await ImagePicker.launchImageLibraryAsync({ + mediaTypes: ImagePicker.MediaTypeOptions.Images, + base64: true, quality: 1, }); if (!result.canceled) { - setImage(result.assets[0].uri); + setImageUri(result.assets[0].uri); + await processImage(result.assets[0].base64); } }; + const takePhoto = async () => { + const hasPermission = await requestCameraPermissions(); + if (!hasPermission) { + setError("Camera permission is required to take a photo."); + return; + } + + const result = await ImagePicker.launchCameraAsync({ + base64: true, + quality: 1, + }); + + if (!result.canceled) { + setImageUri(result.assets[0].uri); + await processImage(result.assets[0].base64); + } + }; + + const processImage = async (base64Image) => { + setLoading(true); + setError(null); + + try { + const response = await fetch( + "https://api.openai.com/v1/chat/completions", + { + method: "POST", + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: "gpt-4o-mini", + messages: [ + { + role: "system", + content: + "Identify and count poker chips by color. Return only the count for each color in JSON format.", + }, + { + role: "user", + content: [ + { + type: "text", + text: "How many poker chips are there for each color? Return structured JSON.", + }, + { + type: "image_url", + image_url: { url: `data:image/png;base64,${base64Image}` }, + }, + ], + }, + ], + max_tokens: 1000, + }), + } + ); + + const result = await response.json(); + + if (!response.ok || !result.choices || !result.choices[0].message) { + throw new Error("Invalid response from API."); + } + + const rawContent = result.choices[0].message.content.trim(); + const cleanJSON = rawContent.replace(/```json|```/g, "").trim(); + + const parsedData = JSON.parse(cleanJSON); + + // Filter out colors with a count of 0 + const filteredData = Object.fromEntries( + Object.entries(parsedData).filter(([_, count]) => count > 0) + ); + + setLastDetectedChips(filteredData); // Store detected chip counts + updateChipCount(filteredData); + } catch (error) { + console.error("Error processing image:", error); + setError("Failed to analyze the image."); + } + + setLoading(false); + }; + return ( - - Automatic Detection - - + +