Breaking News: Grepper is joining You.com. Read the official announcement!

snowflake etl

Add Answer

Innocent Iguana answered on October 2, 2024 Popularity 7/10 Helpfulness 1/10

answer snowflake etl

related pandas json

related javafx drag and drop file

snowflake etl

Comment

# On snowflake, you use ELT instead of ETL
# First you extract data, then you load the data on snowflake, and finally do Transformation inside snowflake

# install necessary packages: pip install snowflake-connector-python pandas
import snowflake.connector


# EXTRACT : Use pd.read_csv, pd.read_parquet, pd.read_json
df = .....

# LOAD : Load the data from your PC (eg: your desktop RDBMS like postgreSQL) to Snowflake account
from sqlalchemy import create_engine

def snowflake_engine(username, password, account_url, warehouse_name, db_name, schema_name):
    engine = create_engine(URL('snowflake',
                                user = username,
                                password = password,
                                account = account_url,
                                warehouse = warehouse_name,
                                database = db_name,
                                schema = schema_name))
    return engine

engine = snowflake_engine('USERNAME', 'PASSWORD', 'ACCOUNTURL', 'WAREHOUSE', 'DATABASE', 'SCHEMA') # Create the engine
df.to_sql('table_name', con=engine, index=False, if_exists='append') # Use the engine to load data into Snowflake

# TRANSFORM : Perform Transformation on warehouse

def warehouse_connection(username, password, account_url, warehouse_name, db_name, schema_name):
    connection = snowflake.connector.connect(
        user = username,
        password = password,
        account = account_url,
        warehouse = warehouse_name,
        database = db_name,
        schema = schema_name
    return connection
)

con = warehouse_connection('USERNAME', 'PASSWORD', 'ACCOUNTURL', 'WAREHOUSE', 'DATABASE', 'SCHEMA') # Establish a connection to the Snowflake data warehouse

def transform(query, connnection=con):
    connnection.cursor().execute(query)


my_query = """
CREATE OR REPLACE TABLE transformed_data AS
SELECT column1, SUM(column2) 
FROM raw_data 
GROUP BY column1;
"""
transform(my_query)

Popularity 7/10 Helpfulness 1/10 Language python

Source: dev.to

Tags: etl whatever

Link to this answer
Share Copy Link

Contributed on Oct 02 2024

Innocent Iguana

0 Answers Avg Quality 2/10

Closely Related Answers

pandas json

Comment

import json

# Loading json file as dictionary
with open("json_file.json", "r") as file:
	raw_stock_data = json.load(file)

# Loading json data as dataframe
import pandas as pd
df = pd.read_json("json_file.json", orient='columns')

# Turning a json into string
json.dumps(json_schema["properties"]) # '{"maximum": 130, "minimum": 1, "type": "integer"}'

# Writing a json file
with open("json_file.json", mode="w") as fh:
	json.dump(obj=json_schema, fp=fh)

### Which orient to use:
# split	-> Separate index, columns, and data as key-value pairs for dataframe -> {"index": [...], "columns": [...], "data": [...]}
# records -> List of dictionaries (each dict is a row) -> [{"col1": value1, "col2": value2}, ...]
# index	-> Dict of dicts (rows are keys) -> {"row1": {"col1": value1, "col2": value2}, ...}
# columns -> Dict of lists (columns are keys) -> {"col1": [values], "col2": [values], ...}
# values ->	List of lists (each list is a row) -> [[value1, value2], [value3, value4]]

Popularity 9/10 Helpfulness 5/10 Language python

Source: Grepper

Tags: python pandas json

Link to this answer
Share Copy Link

Contributed on Oct 02 2024

Innocent Iguana

0 Answers Avg Quality 2/10

javafx drag and drop file

Comment

public class DragAndDropTest extends Application {

    @Override
    public void start(Stage primaryStage) {
        Label label = new Label("Drag a file to me.");
        Label dropped = new Label("");
        VBox dragTarget = new VBox();
        dragTarget.getChildren().addAll(label,dropped);
        dragTarget.setOnDragOver(new EventHandler<DragEvent>() {

            @Override
            public void handle(DragEvent event) {
                if (event.getGestureSource() != dragTarget
                        && event.getDragboard().hasFiles()) {
                    /* allow for both copying and moving, whatever user chooses */
                    event.acceptTransferModes(TransferMode.COPY_OR_MOVE);
                }
                event.consume();
            }
        });

        dragTarget.setOnDragDropped(new EventHandler<DragEvent>() {

            @Override
            public void handle(DragEvent event) {
                Dragboard db = event.getDragboard();
                boolean success = false;
                if (db.hasFiles()) {
                    dropped.setText(db.getFiles().toString());
                    success = true;
                }
                /* let the source know whether the string was successfully 
                 * transferred and used */
                event.setDropCompleted(success);

                event.consume();
            }
        });


        StackPane root = new StackPane();
        root.getChildren().add(dragTarget);

        Scene scene = new Scene(root, 500, 250);

        primaryStage.setTitle("Drag Test");
        primaryStage.setScene(scene);
        primaryStage.show();
    }

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        launch(args);
    }

}

Popularity 3/10 Helpfulness 1/10 Language java

Source: stackoverflow.com

Tags: javafx drag-and-drop java file

Link to this answer
Share Copy Link

Contributed on Oct 02 2024

NG CHEE WEI

0 Answers Avg Quality 2/10

snowflake etl

Contents

More Related Answers

snowflake etl

Closely Related Answers

pandas json

javafx drag and drop file

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.