[Get reference pictures from preview to learn a neural network]

Hi, Thx for you answers.

I have started to try to at least get the cura plugin to start and I´ve gotten so far that I cover the basic Menu item, however I copied the code from the UFP writer and tried to get the method rolling, but I´m stuck at understanding how to get the stream into the method, since I want to to run the script from the menu item I guess I need to add something by the start to access the stream. As you´ll see below I´m pretty novice on this - the out commented Write method need the input stream.

 

from typing import List
from typing import cast
from Charon.VirtualFile import VirtualFile #To open UFP files.
from Charon.OpenMode import OpenMode #To indicate that we want to write to UFP files.
from io import StringIO #For converting g-code to bytes.
from UM.Application import Application
from UM.Logger import Logger
from UM.Mesh.MeshWriter import MeshWriter #The writer we need to implement.
from UM.MimeTypeDatabase import MimeTypeDatabase, MimeType
from UM.PluginRegistry import PluginRegistry #To get the g-code writer.
from PyQt5.QtCore import QBuffer
from cura.Snapshot import Snapshot
from cura.Utils.Threading import call_on_qt_thread
from UM.i18n import i18nCatalog
from UM.Extension import Extension
from UM.PluginRegistry import PluginRegistry
from UM.Scene.SceneNode import SceneNode
from UM.Scene.Selection import Selection
from UM.Message import Message
from cura.CuraApplication import CuraApplication
import os

i18n_catalog = i18nCatalog("CreateTrainingDataPlugin")

class CreateTrainingDataPlugin(Extension, MeshWriter):
    def __init__(self):
        super().__init__()
        self.addMenuItem(i18n_catalog.i18n("Create training data pictures"), self.doExtendedCreateTrainingPics)
        #self.addMenuItem(i18n_catalog.i18n("Write"), self.doExtendedWrite)

        self._message = None
        MimeTypeDatabase.addMimeType(
            MimeType(
                name = "application/x-ufp",
                comment = "Ultimaker Format Package",
                suffixes = ["ufp"]
            )
        )
        self._snapshot = None

    def doExtendedCreateTrainingPics(self):
        self.doCreateTrainingdata(True)
    
    #@call_on_qt_thread
    #def doExtendedWrite(self):
    #   self.write(stream)
    @call_on_qt_thread
    def doCreateTrainingdata(self, extended_mode):
        self._message = Message(i18n_catalog.i18nc("@info:status", "Creating .PNG pics"), title = i18n_catalog.i18nc("@title", "PNG Pics"))
        self._message.show()
        #self.write()

    def _createSnapshot(self, *args):
        # must be called from the main thread because of OpenGL
        Logger.log("d", "Creating thumbnail image...")
        try:
            self._snapshot = Snapshot.snapshot(width = 300, height = 300)
        except Exception:
            Logger.logException("w", "Failed to create snapshot image")
            self._snapshot = None

    @call_on_qt_thread
    def write(self, stream, nodes, mode = MeshWriter.OutputMode.BinaryMode):
        archive = VirtualFile()
        archive.openStream(stream, "application/x-ufp", OpenMode.WriteOnly)

        #Store the g-code from the scene.
        archive.addContentType(extension = "gcode", mime_type = "text/x-gcode")
        gcode_textio = StringIO() #We have to convert the g-code into bytes.
        gcode_writer = cast(MeshWriter, PluginRegistry.getInstance().getPluginObject("GCodeWriter"))
        success = gcode_writer.write(gcode_textio, None)
        if not success: #Writing the g-code failed. Then I can also not write the gzipped g-code.
            self.setInformation(gcode_writer.getInformation())
            return False
        gcode = archive.getStream("/3D/model.gcode")
        gcode.write(gcode_textio.getvalue().encode("UTF-8"))
        archive.addRelation(virtual_path = "/3D/model.gcode", relation_type = "http://schemas.ultimaker.org/package/2018/relationships/gcode")

        self._createSnapshot()

        #Store the thumbnail.
        if self._snapshot:
            archive.addContentType(extension = "png", mime_type = "image/png")
            thumbnail = archive.getStream("/Metadata/thumbnail.png")

            thumbnail_buffer = QBuffer()
            thumbnail_buffer.open(QBuffer.ReadWrite)
            thumbnail_image = self._snapshot
            thumbnail_image.save(thumbnail_buffer, "PNG")

            thumbnail.write(thumbnail_buffer.data())
            archive.addRelation(virtual_path = "/Metadata/thumbnail.png", relation_type = "http://schemas.openxmlformats.org/package/2006/relationships/metadata/thumbnail", origin = "/3D/model.gcode")
        else:
            Logger.log("d", "Thumbnail not created, cannot save it")

 

[Get reference pictures from preview to learn a neural network]

Hi,

I’m working on a project where I have trained a neural network model ”CNN” with data from the preview in Cura. The CNN are capable of predicting the outcom of the print - pass failed for each layer, and I’d like to write a plugin to Cura where I’d get pictures from a couple of angles from each layer in the preview. These pictures will then be sent via SSH to the SBC ”Jetson Nano” having duet 3 board connected. This will enable the possibillty to verify the print on the fly. So before I dive into the Plugin writing I’m wondering if it’s even possible to generate pictures using code in a plugin?