From 15c1f9db9241c7f10cb4fcc12bd96868686ca7ae Mon Sep 17 00:00:00 2001 From: akhatre <34001573+akhatre@users.noreply.github.com> Date: Fri, 16 Apr 2021 17:14:01 +0600 Subject: [PATCH] Add lookml-gen (#33) * proof of concept implementation * cleaned up the code, added more lookml features * upd version and readme * remove binary * removed supportal linking * add more error handling and changed formatting * incremented version Co-authored-by: Akhat Rakishev --- .gitignore | 3 + README.md | 1 + cmd/lookml_gen.go | 137 ++++++++++++++++++++++++++++++++++++++++++++++ utils/version.go | 2 +- 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 cmd/lookml_gen.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fbdadb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/ddbt_dev +/.DS_Store +/.idea \ No newline at end of file diff --git a/README.md b/README.md index 7cb18c0..32223c2 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ ddbt version 0.2.1 - `ddbt completion zsh` will generate a shell completion script zsh (or bash if you pass that as argument). Detailed steps to set up the completion script can be found in `ddbt completion --help` - `ddbt isolate-dag` will create a temporary directory and symlink in all files needed for the given _model_filter_ such that Fishtown's DBT could be run against it without having to be run against every model in your data warehouse - `ddbt schema-gen -m my_model` will output a new or updated schema yml file for the model provided in the same directory as the dbt model file. +- `ddbt lookml-gen my_model` will generate lookml view and copy it to your clipboard ### Global Arguments - `--models model_filter` _or_ `-m model_filter`: Instead of running for every model in your project, DDBT will only execute against the requested models. See filters below for what is accepted for `my_model` diff --git a/cmd/lookml_gen.go b/cmd/lookml_gen.go new file mode 100644 index 0000000..b1b2550 --- /dev/null +++ b/cmd/lookml_gen.go @@ -0,0 +1,137 @@ +package cmd + +import ( + "ddbt/bigquery" + "ddbt/config" + "errors" + + "fmt" + "os" + + "github.com/atotto/clipboard" + "github.com/spf13/cobra" +) + +// map bigquery data types to looker data types +var mapBqToLookerDtypes map[string]string = map[string]string{ + "INTEGER": "number", + "FLOAT": "number", + "NUMERIC": "number", + "BOOLEAN": "yesno", + "STRING": "string", + "TIMESTAMP": "time", + "DATETIME": "time", + "DATE": "time", + "TIME": "time", + "BOOL": "yesno", + "ARRAY": "string", + "GEOGRAPHY": "string", +} + +// specify looker timeframes for datetime/date/time variable data types +const timeBlock string = `timeframes: [ + raw, + time, + date, + week, + month, + quarter, + year +] +` + +func init() { + rootCmd.AddCommand(lookmlGenCmd) +} + +var lookmlGenCmd = &cobra.Command{ + Use: "lookml-gen [model name]", + Short: "Generates the .view.lkml file for a given model", + Args: cobra.ExactValidArgs(1), + ValidArgsFunction: completeModelFn, + Run: func(cmd *cobra.Command, args []string) { + modelName := args[0] + + // get filesystem, model and target + fileSystem, _ := compileAllModels() + model := fileSystem.Model(modelName) + + target, err := model.GetTarget() + if err != nil { + fmt.Println("Could not get target for schema") + os.Exit(1) + } + fmt.Println("\nšŸŽÆ Target for retrieving schema:", target.ProjectID+"."+target.DataSet) + + // generate lookml view + err = generateNewLookmlView(modelName, target) + + if err != nil { + fmt.Println("šŸ˜’ Something went wrong at lookml view generation: ", err) + os.Exit(1) + } + + }, +} + +func getColumnsForModelWithDtypes(modelName string, target *config.Target) (columns []string, dtypes []string, err error) { + schema, err := bigquery.GetColumnsFromTable(modelName, target) + if err != nil { + fmt.Println("Could not retrieve schema from BigQuery") + os.Exit(1) + } + + // itereate over fields, record field names and data types + for _, fieldSchema := range schema { + columns = append(columns, fieldSchema.Name) + dtypes = append(dtypes, string(fieldSchema.Type)) + } + return columns, dtypes, err +} + +func generateNewLookmlView(modelName string, target *config.Target) error { + bqColumns, bqDtypes, err := getColumnsForModelWithDtypes(modelName, target) + if err != nil { + fmt.Println("Retrieved BigQuery schema but failed to parse it") + os.Exit(1) + } + + // initialise lookml view head + lookmlView := "view: " + modelName + " {\n\n" + lookmlView += "sql_table_name: `" + target.ProjectID + "." + target.DataSet + "." + modelName + "` ;;\n" + + // add dimensions and appropriate blocks for each field + for i := 0; i < len(bqColumns); i++ { + colName := bqColumns[i] + colDtype := mapBqToLookerDtypes[bqDtypes[i]] + if colDtype == "" { + return errors.New("Did not find Looker data type corresponding to BigQuery data type: " + bqDtypes[i]) + } + newBlock := "\n" + + if colDtype == "date_time" || colDtype == "date" || colDtype == "time" { + newBlock += "dimension_group: " + colName + " {\n" + newBlock += "type: " + colDtype + "\n" + newBlock += timeBlock + } else { + newBlock += "dimension: " + colName + " {\n" + newBlock += "type: " + colDtype + "\n" + } + + newBlock += "sql: ${TABLE}." + colName + " ;;\n}\n" + + lookmlView += newBlock + } + + // add closing curly bracket and copy to clipboard + lookmlView += "}" + + err = clipboard.WriteAll(lookmlView) + if err != nil { + fmt.Println("Could not write generated LookML to your clipboard") + os.Exit(1) + } + fmt.Println("\nāœ… LookML view for " + modelName + " has been copied to your clipboard!") + + return nil +} diff --git a/utils/version.go b/utils/version.go index e9acf90..daa6823 100644 --- a/utils/version.go +++ b/utils/version.go @@ -1,3 +1,3 @@ package utils -const DdbtVersion = "0.4.6" +const DdbtVersion = "0.5.0"