{"allowedRenditionsWidth":["320","480","640","768","960","1200","1440","1920"],"templateName":"quickstart-page-template","cssClassNames":"page basicpage summit-page","canonicalLink":"https://www.snowflake.com/en/developers/guides/snowflake-postgres-stream-snowflake-with-pglake/","robotsTags":[],"language":"en","title":"Stream Snowflake Changes to Postgres with pg_lake","analyticsPageType":"homepage","analyticsCategory":"general","analyticsSubCategory":"","excludeFromAnalytics":false,":mappedPath":"/en/developers/guides/snowflake-postgres-stream-snowflake-with-pglake/",":type":"snowflake-site/components/structure/page",":items":{"root":{"columnCount":12,"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","columnClassNames":{"markup_editor_1950346551":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-banner":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-header":"aem-GridColumn aem-GridColumn--default--12","responsivegrid":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-footer":"aem-GridColumn aem-GridColumn--default--12","modal_container":"aem-GridColumn aem-GridColumn--default--12","markup_editor":"aem-GridColumn aem-GridColumn--default--12"},":items":{"experiencefragment-banner":{"id":"experiencefragment-99e21cd2d7","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/pushdown-banner/master/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/pushdown-banner/master.xfmodel.json"},"experiencefragment-header":{"id":"experiencefragment-162cf0ae17","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/mega-nav-header/master/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/mega-nav-header/master.xfmodel.json","languageNavPath":"/content/snowflake-site/global/en/developers/guides/snowflake-postgres-stream-snowflake-with-pglake.languagenav.json"},"markup_editor_1950346551":{"id":"markup-editor-8132918268","title":" ","cssContent":".snowflake-markdown-table code[class*=language-],.snowflake-markdown-table code[class*=language-],.snowflake-markdown .snowflake-text code[class*=language-],.snowflake-markdown .snowflake-text pre[class*=language-]{background-color:rgba(var(--ui-12-rgb),.5);color:var(--text-01);text-shadow:none;padding:var(--spacing-00);border-radius:var(--spacing-00);font-size:smaller}","isGSAPEnabled":false,":type":"snowflake-site/components/markup-editor"},"responsivegrid":{"columnCount":12,"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","columnClassNames":{"quickstart_hero":"aem-GridColumn aem-GridColumn--default--12","flexible_column_cont":"aem-GridColumn aem-GridColumn--default--12","markup_editor":"aem-GridColumn aem-GridColumn--default--12"},":items":{"quickstart_hero":{"id":"quickstart-hero-15168565eb","fragmentPath":"/content/dam/snowflake-site/en/content-fragments/quickstarts/snowflake-postgres-stream-snowflake-with-pglake",":type":"snowflake-site/components/quickstart/quickstart-hero","isDeveloperGuidesPage":false,"quickstartHeroFirstCertifiedTag":{"tagText":"Quickstart","tagColor":"#29B5E8","tagPath":"/content/cq:tags/snowflake-site/taxonomy/solution-center/certification/quickstart","tagIcon":""},"quickstartHeroTitle":{"lines":["Stream Snowflake Changes to Postgres with pg_lake"],"type":"heading2",":type":"snowflake-site/components/title-v2"},"quickstartHeroAuthor":"Brian Pace","quickstartHeroForkRepoLink":{"id":"button-f19cfe3262","showOutboundIcon":false,"buttonLink":{"valid":true,"attributes":{"target":"_blank"},"url":"https://github.com/Snowflake-Labs/sfquickstarts/tree/master/site/sfguides/src/snowflake-postgres-stream-snowflake-with-pglake"},"linkTargetContentType":"GENERIC",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_EXTERNAL","text":"Fork Repo"},"quickstartHeroBreadcrumbs":[{"title":"Stream Snowflake Changes to Postgres with pg_lake","url":"https://www.snowflake.com/content/snowflake-site/global/en/developers/guides/snowflake-postgres-stream-snowflake-with-pglake","currentPage":true},{"title":"Guides","url":"https://www.snowflake.com/content/snowflake-site/global/en/developers/guides","currentPage":false},{"title":"Snowflake for Developers","url":"https://www.snowflake.com/content/snowflake-site/global/en/developers","currentPage":false}]},"flexible_column_cont":{"id":"flexible-column-container-6692b2c648","propertiesId":"quickstart-template-main-flexible-container","type":"2-column-75-25","alignColumns":"top","containerMaxWidth":"extra-large","topPadding":"none","bottomPadding":"none","spaceBetween":"small","reverseOnMobile":false,"carouselOnMobile":false,"backgroundImageOption":"none","flexible_column_content_container_1":{"layout":"SIMPLE","id":"container-6cedfd9508",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{"contentfragment":{"id":"contentfragment-f21342fffd","paragraphs":["<!-- ------------------------ -->\n","\u003Ch2\u003EOverview\u003C/h2\u003E\n","\u003Ch3\u003EWhat You'll Build\u003C/h3\u003E\n","\u003Cp\u003EThis quickstart demonstrates how to build an automated pipeline that streams row-level changes (INSERT, UPDATE, DELETE) from a Snowflake table down to a Snowflake Postgres table. Changes made in Snowflake are exported as Parquet files to a shared internal stage by a scheduled Task, then detected and merged into Postgres by a \u003Ccode\u003Epg_incremental\u003C/code\u003E pipeline — with no S3 bucket, no external pipeline, and no manual intervention.\u003C/p\u003E\n","\u003Ch3\u003EWhat You'll Learn\u003C/h3\u003E\n\u003Cul\u003E\u003Cli\u003EHow Snowflake Streams present captured changes\u003C/li\u003E\u003Cli\u003EEasy way to setup change data capture from Snowflake to Postgres\u003C/li\u003E\u003Cli\u003EAutomate file loads with pg_incremental\u003C/li\u003E\u003Cli\u003EManaging floating point numbers when exporting to parquet\u003C/li\u003E\u003C/ul\u003E\n","\u003Ch3\u003EWhat You'll Need\u003C/h3\u003E\n\u003Cul\u003E\u003Cli\u003ESnowflake account with ACCOUNTADMIN access\u003C/li\u003E\u003Cli\u003ELocal terminal session to run \u003Ccode\u003Epsql\u003C/code\u003E\u003C/li\u003E\u003Cli\u003E\u003Ccode\u003Epsql\u003C/code\u003E client installed locally (install with \u003Ccode\u003Ebrew install postgresql\u003C/code\u003E on macOS)\u003C/li\u003E\u003Cli\u003EFamiliarity with SQL and basic Postgres concepts\u003C/li\u003E\u003C/ul\u003E\n","\u003Ch3\u003EHow It Works\u003C/h3\u003E\n","\u003Cp\u003ESnowflake Streams capture every row-level change made to a table since the stream was last consumed. A scheduled Snowflake Task reads the stream and writes the captured changes as Parquet files to a shared internal stage. On the Postgres side, \u003Ccode\u003Epg_incremental\u003C/code\u003E polls the stage every 30 seconds and calls a sync function for each new file. The sync function reads the Parquet file using a \u003Ccode\u003Epg_lake\u003C/code\u003E foreign table and merges each record into the local Postgres table.\u003C/p\u003E\n","\u003Ch3\u003EArchitecture\u003C/h3\u003E\n","\u003Cp\u003E\u003Cimg src=\"https://www.snowflake.com/content/dam/snowflake-site/developers/guides/snowflake-postgres-stream-snowflake-with-pglake/architecture.png\" alt=\"Architecture\"\u003E\u003C/p\u003E\n","\u003Ch3\u003EStream Change Record Semantics\u003C/h3\u003E\n","\u003Cp\u003ESnowflake streams encode all three change types using two metadata columns:\u003C/p\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003E_action\u003C/code\u003E\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003E_is_update\u003C/code\u003E\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EMeaning\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EPostgres action\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EINSERT\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EFALSE\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ENew row inserted\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EINSERT\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EINSERT\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003ETRUE\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ENew version of an updated row\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EINSERT\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EDELETE\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EFALSE\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ERow deleted\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EDELETE\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EDELETE\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003ETRUE\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EOld version of an updated row\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EDELETE\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Cp\u003EAn UPDATE produces two stream records for the same \u003Ccode\u003Eproduct_id\u003C/code\u003E — a \u003Ccode\u003EDELETE\u003C/code\u003E (old values) and an \u003Ccode\u003EINSERT\u003C/code\u003E (new values), both with \u003Ccode\u003E_is_update=TRUE\u003C/code\u003E. Crucially, a Snowflake stream reports the \u003Cstrong\u003Enet outcome\u003C/strong\u003E of each row between two consumes, not a full statement-by-statement history: within a single exported file, each \u003Ccode\u003Eproduct_id\u003C/code\u003E appears as at most a single \u003Ccode\u003EINSERT\u003C/code\u003E, a single \u003Ccode\u003EDELETE\u003C/code\u003E, or one \u003Ccode\u003EDELETE\u003C/code\u003E+\u003Ccode\u003EINSERT\u003C/code\u003E update pair. The sync function uses this guarantee to apply each file with a single set-based \u003Ccode\u003EMERGE\u003C/code\u003E: the \u003Ccode\u003E_is_update\u003C/code\u003E flag distinguishes a genuine deletion (\u003Ccode\u003E_is_update=FALSE\u003C/code\u003E) from the delete-half of an update (\u003Ccode\u003E_is_update=TRUE\u003C/code\u003E), so after filtering out the update-half deletes there is exactly one intended action per key. This makes UPDATE handling completely transparent — no row-by-row loop and no special update-detection logic is needed.\u003C/p\u003E\n","\u003Ch3\u003EFile Ordering Guarantee\u003C/h3\u003E\n","\u003Cp\u003EWith a scheduled task writing files over time, Postgres must apply files in the order they were written to preserve data integrity. This is handled in two layers:\u003C/p\u003E\n","\u003Cp\u003E\u003Cstrong\u003ECross-run ordering\u003C/strong\u003E is guaranteed by one property and enforced by a sorted wrapper function:\u003C/p\u003E\n\u003Col\u003E\u003Cli\u003E\u003Cstrong\u003ESnowflake tasks run serially.\u003C/strong\u003E A task never executes concurrently with itself. Each run atomically consumes the full stream snapshot and writes its file before the next run can start. All files from task run N therefore have an earlier modification timestamp than any file from task run N+1.\u003C/li\u003E\u003Cli\u003E\u003Cstrong\u003E\u003Ccode\u003Elake_file.list()\u003C/code\u003E does not sort its results.\u003C/strong\u003E It passes through whatever order the object storage API returns with no sort step. \u003Ccode\u003Epg_incremental\u003C/code\u003E also applies no sort — it builds an internal list in whatever row order the list function returns and processes files in that order. To guarantee modification-time ordering, this quickstart passes a \u003Ccode\u003Epublic.list_files_sorted\u003C/code\u003E wrapper function (created in Postgres Setup Step 5) that adds \u003Ccode\u003EORDER BY last_modified_time ASC\u003C/code\u003E. Without this, file order would be undefined and older files could be applied after newer ones.\u003C/li\u003E\u003C/ol\u003E\n","\u003Cp\u003E\u003Cstrong\u003EWithin-run ordering\u003C/strong\u003E is handled by using \u003Ccode\u003ESINGLE = TRUE\u003C/code\u003E in the \u003Ccode\u003ECOPY INTO\u003C/code\u003E statement. This forces all stream records from a single task run into one file. Without this, the \u003Ccode\u003ECOPY INTO\u003C/code\u003E may split records across multiple files with no guaranteed distribution order — meaning the DELETE and INSERT halves of the same UPDATE pair could land in different files and be applied out of order, producing incorrect results. With \u003Ccode\u003ESINGLE = TRUE\u003C/code\u003E, each task run produces exactly one file containing a complete, self-consistent batch, which the sync function processes safely in a single transaction.\u003C/p\u003E\n","\u003Ch3\u003EData Model\u003C/h3\u003E\n","\u003Cp\u003E\u003Cstrong\u003ESnowflake \u003Ccode\u003Eproducts\u003C/code\u003E table:\u003C/strong\u003E\u003C/p\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EColumn\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EType\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EDescription\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Eproduct_id\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EINT\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EPrimary key\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Eproduct_name\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EVARCHAR(100)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EDisplay name\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Ecategory\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EVARCHAR(50)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EProduct category\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Eprice\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ENUMBER(10,2)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ECurrent price\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Estatus\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EVARCHAR(20)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Eactive\u003C/code\u003E or \u003Ccode\u003Ediscontinued\u003C/code\u003E\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Eupdated_at\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ETIMESTAMP_NTZ\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ELast modified in Snowflake\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Cp\u003E\u003Cstrong\u003EPostgres \u003Ccode\u003Eproducts\u003C/code\u003E table\u003C/strong\u003E — same schema plus:\u003C/p\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EColumn\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EType\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EDescription\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Esynced_at\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ETIMESTAMP\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EWhen the row was last written by the sync pipeline\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Ch3\u003EPrerequisites\u003C/h3\u003E\n\u003Cul\u003E\u003Cli\u003ESnowflake account with ACCOUNTADMIN access\u003C/li\u003E\u003Cli\u003ELocal terminal session to run \u003Ccode\u003Epsql\u003C/code\u003E\u003C/li\u003E\u003Cli\u003EFamiliarity with SQL and basic Postgres concepts\u003C/li\u003E\u003C/ul\u003E\n<!-- ------------------------ -->\n","\u003Ch2\u003EPostgres Instance Setup\u003C/h2\u003E\n","\u003Cp\u003ECreate a Snowflake Postgres instance. Skip this section if you already have an instance you want to use — just substitute its name for \u003Ccode\u003EPG_LAB\u003C/code\u003E throughout.\u003C/p\u003E\n","\u003Ch3\u003EStep 1: Create Network Policy\u003C/h3\u003E\n","\u003Cp\u003ESnowflake Postgres requires a network policy to allow client connections. Replace \u003Ccode\u003Ennn.nnn.nnn.nnn/32\u003C/code\u003E with a specific IP address or CIDR for your organization.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Instance Setup: Step 1 - Create Network Policy\n-- Execute in: Snowsight (Snowflake)\nUSE ROLE ACCOUNTADMIN;\n\nCREATE DATABASE IF NOT EXISTS pg_network_db;\nCREATE SCHEMA  IF NOT EXISTS pg_network_db.pg_network;\n\nUSE SCHEMA pg_network_db.pg_network;\n\n-- Run this to find your current IP address if needed\nSELECT current_ip_address();\n\nCREATE OR REPLACE NETWORK RULE pg_lab_ingress_rule\n    TYPE       = IPV4\n    VALUE_LIST = ('nnn.nnn.nnn.nnn/32')\n    MODE       = POSTGRES_INGRESS\n    COMMENT    = 'Allow Postgres client connections (restrict in production)';\n\nCREATE OR REPLACE NETWORK POLICY pg_lab_network_policy\n    ALLOWED_NETWORK_RULE_LIST = ('pg_lab_ingress_rule')\n    COMMENT = 'Network policy for Snowflake Postgres instances';\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 2: Create Postgres Instance\u003C/h3\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Instance Setup: Step 2 - Create Postgres Instance\n-- Execute in: Snowsight (Snowflake)\n\nCREATE POSTGRES INSTANCE PG_LAB\n    AUTHENTICATION_AUTHORITY = POSTGRES_OR_SNOWFLAKE\n    COMPUTE_FAMILY           = 'STANDARD_L'\n    STORAGE_SIZE_GB          = 50\n    POSTGRES_VERSION         = 18\n    HIGH_AVAILABILITY        = FALSE\n    NETWORK_POLICY           = 'pg_lab_network_policy'\n    COMMENT                  = 'Postgres instance for stream-to-postgres quickstart';\n\u003C/code\u003E\u003C/pre\u003E\n\u003Cblockquote\u003E\n","\u003Cp\u003E\u003Cstrong\u003EImportant:\u003C/strong\u003E Save the username and password displayed in the \u003Ccode\u003Eaccess_roles\u003C/code\u003E field and the \u003Ccode\u003Ehost\u003C/code\u003E value — you will need them to connect.\u003C/p\u003E\n\u003C/blockquote\u003E\n","\u003Ch3\u003EStep 3: Monitor Instance Status\u003C/h3\u003E\n","\u003Cp\u003EWait for the instance to reach READY state before proceeding.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Instance Setup: Step 3 - Monitor Instance Status\n-- Execute in: Snowsight (Snowflake)\n\nSHOW POSTGRES INSTANCES;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 4: Configure psql Connection\u003C/h3\u003E\n","\u003Cp\u003ESet environment variables in your terminal once the instance is READY:\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-bash\"\u003Eexport PGHOST=<hostname from SHOW POSTGRES INSTANCES>\nexport PGPORT=5432\nexport PGDATABASE=postgres\nexport PGUSER=snowflake_admin\nexport PGPASSWORD=<password from CREATE POSTGRES INSTANCE output>\nexport PGSSLMODE=require\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003ETest the connection:\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-bash\"\u003Epsql -c "SELECT version();"\n\u003C/code\u003E\u003C/pre\u003E\n<!-- ------------------------ -->\n","\u003Ch2\u003ESnowflake Setup\u003C/h2\u003E\n","\u003Cp\u003ECreate the Snowflake database, table, stage, stream, and export task.\u003C/p\u003E\n\u003Cblockquote\u003E\n","\u003Cp\u003E\u003Cstrong\u003ERole Required:\u003C/strong\u003E This section requires \u003Cstrong\u003EACCOUNTADMIN\u003C/strong\u003E to create the storage integration.\u003C/p\u003E\n\u003C/blockquote\u003E\n","\u003Ch3\u003EStep 1: Create Database, Schema, Products Table, and Sample Data\u003C/h3\u003E\n","\u003Cp\u003EThis simulates a realistic starting point where a table already exists and contains data before the sync pipeline is set up.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Snowflake Setup: Step 1 - Create Database, Schema, Products Table, and Sample Data\n-- Execute in: Snowsight (Snowflake)\n\nCREATE DATABASE IF NOT EXISTS stream_lab;\nCREATE SCHEMA  IF NOT EXISTS stream_lab.catalog;\n\nUSE SCHEMA stream_lab.catalog;\n\nCREATE OR REPLACE TABLE products (\n    product_id    INT            NOT NULL,\n    product_name  VARCHAR(100)   NOT NULL,\n    category      VARCHAR(50)    NOT NULL,\n    price         NUMBER(10,2)   NOT NULL,\n    status        VARCHAR(20)    NOT NULL,\n    updated_at    TIMESTAMP_NTZ  DEFAULT current_timestamp(),  -- must be set on every INSERT and UPDATE; used for sync ordering\n    PRIMARY KEY (product_id)\n);\n\nINSERT INTO products (product_id, product_name, category, price, status)\nVALUES\n    (1, 'Wireless Headphones',   'electronics',  89.99, 'active'),\n    (2, 'USB-C Hub 7-Port',      'electronics',  49.99, 'active'),\n    (3, 'Mechanical Keyboard',   'electronics', 129.99, 'active'),\n    (4, 'Running Jacket',        'apparel',       74.99, 'active'),\n    (5, 'Merino Wool Socks',     'apparel',       18.99, 'active'),\n    (6, 'Trail Running Shoes',   'apparel',      119.99, 'active'),\n    (7, 'Bamboo Desk Organizer', 'home',          34.99, 'active'),\n    (8, 'LED Desk Lamp',         'home',          55.99, 'active');\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 2: Create Storage Integration, Stage, and Verify\u003C/h3\u003E\n","\u003Cp\u003EThe storage integration links the Snowflake internal stage to the Postgres instance's internal storage so both sides can read and write the same files.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Snowflake Setup: Step 2 - Create Storage Integration, Stage, and Verify\n-- Execute in: Snowsight (Snowflake)\nCREATE OR REPLACE STORAGE INTEGRATION pg_lab_stream_int\n    TYPE              = POSTGRES_INTERNAL_STORAGE\n    POSTGRES_INSTANCE = 'PG_LAB'\n    ENABLED           = TRUE;\n\nCREATE OR REPLACE STAGE stream_sync_stage\n    STORAGE_INTEGRATION = pg_lab_stream_int\n    RELATIVE_URL        = '/';\n\n-- Confirm stage is accessible (returns empty result if no files yet)\nLIST @stream_sync_stage/;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 3: Create Stream and View Initial Rows\u003C/h3\u003E\n","\u003Cp\u003E\u003Ccode\u003ESHOW_INITIAL_ROWS = TRUE\u003C/code\u003E causes the stream to surface all existing rows as INSERT records the first time it is consumed, regardless of when those rows were inserted. This is the recommended approach when setting up sync against a table that already contains data — no separate initial-load step is needed.\u003C/p\u003E\n","\u003Cp\u003E\u003Ccode\u003EAPPEND_ONLY = FALSE\u003C/code\u003E captures INSERT, UPDATE, and DELETE operations going forward. Each UPDATE produces two stream records: a DELETE record (old values) and an INSERT record (new values).\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Snowflake Setup: Step 3 - Create Stream and View Initial Rows\n-- Execute in: Snowsight (Snowflake)\nCREATE OR REPLACE STREAM products_stream\n    ON TABLE products\n    APPEND_ONLY       = FALSE\n    SHOW_INITIAL_ROWS = TRUE\n    COMMENT = 'Captures all changes to the products table for sync to Postgres';\n\n-- All 8 existing rows should appear with _action = 'INSERT', _is_update = FALSE\nSELECT\n    METADATA$ACTION   AS _action,\n    METADATA$ISUPDATE AS _is_update,\n    product_id, product_name, category, price, status\nFROM products_stream\nORDER BY product_id;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 4: Create Export Procedure, Task, and Verify\u003C/h3\u003E\n","\u003Cp\u003E\u003Ccode\u003ESINGLE = TRUE\u003C/code\u003E and \u003Ccode\u003EINCLUDE_QUERY_ID = TRUE\u003C/code\u003E cannot be used together — Snowflake raises a compilation error. The solution is a stored procedure that builds a timestamp-based suffix using \u003Ccode\u003EEXECUTE IMMEDIATE\u003C/code\u003E, so each task run writes its single file with a unique time based suffix (e.g. \u003Ccode\u003Eproducts/product_stream_20240115_103000.parquet\u003C/code\u003E). The task then calls the procedure instead of running \u003Ccode\u003ECOPY INTO\u003C/code\u003E directly.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Snowflake Setup: Step 4 - Create Export Procedure, Task, and Verify\n-- Execute in: Snowsight (Snowflake)\nCREATE OR REPLACE PROCEDURE export_products_stream()\nRETURNS STRING\nLANGUAGE SQL\nEXECUTE AS CALLER\nAS\n$$\nDECLARE\n    stage_path TEXT;\n    row_count  INTEGER;\nBEGIN\n    row_count := (SELECT COUNT(*) FROM stream_lab.catalog.products_stream);\n\n    IF (row_count = 0) THEN\n        RETURN NULL;\n    END IF;\n\n    stage_path := '@stream_sync_stage/products/product_stream_'\n                  || TO_CHAR(CURRENT_TIMESTAMP(), 'YYYYMMDD_HH24MISS')\n                  || '.parquet';\n\n    EXECUTE IMMEDIATE\n        'COPY INTO ' || :stage_path || '\n        FROM (\n            SELECT\n                product_id,\n                product_name,\n                category,\n                ROUND(price * 100)::INTEGER AS price_cents,\n                status,\n                updated_at,\n                METADATA$ACTION            AS _action,\n                METADATA$ISUPDATE::BOOLEAN AS _is_update\n            FROM stream_lab.catalog.products_stream\n        )\n        FILE_FORMAT = (TYPE = ''PARQUET'')\n        SINGLE      = TRUE';\n\n    RETURN :stage_path || ' (' || :row_count::TEXT || ' rows)';\nEND;\n$$;\n\nCREATE OR REPLACE TASK export_products_stream_task\n    WAREHOUSE = COMPUTE_WH\n    SCHEDULE  = '1 MINUTE'\nWHEN\n    SYSTEM$STREAM_HAS_DATA('stream_lab.catalog.products_stream')\nAS\n    CALL export_products_stream();\n\nALTER TASK export_products_stream_task RESUME;\n\n-- Wait up to 1 minute for the task to run on schedule, then verify\n-- Files land at products/product_stream_YYYYMMDD_HH24MISS.parquet\nLIST @stream_sync_stage/products/;\n\n-- Confirm the stream is now empty (task consumed the initial rows)\nSELECT count(*) AS pending_changes FROM products_stream;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003EWait up to 1 minute for the task to run. You should see a Parquet file containing a timestamp in the file name and \u003Ccode\u003Epending_changes = 0\u003C/code\u003E.\u003C/p\u003E\n<!-- ------------------------ -->\n","\u003Ch2\u003EPostgres Setup\u003C/h2\u003E\n","\u003Cp\u003EInstall extensions, create the products table, create the sync function, and start the \u003Ccode\u003Epg_incremental\u003C/code\u003E pipeline.\u003C/p\u003E\n","\u003Ch3\u003EStep 1: Enable Extensions\u003C/h3\u003E\n","\u003Cp\u003E\u003Ca href=\"https://www.postgresql.org/docs/current/app-psql.html\"\u003Epsql\u003C/a\u003E is the interactive terminal for PostgreSQL, allowing you to enter queries, execute SQL commands, and manage the database from the command line.\u003C/p\u003E\n","\u003Cp\u003EConnect to Postgres via \u003Ccode\u003Epsql\u003C/code\u003E and enable the required extensions (assumes you set the PG* environment variables from the earlier step):\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-bash\"\u003Epsql\n\u003C/code\u003E\u003C/pre\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 1 - Enable Extensions\n-- Execute in: psql (Postgres)\nCREATE EXTENSION IF NOT EXISTS pg_lake CASCADE;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\nCREATE EXTENSION IF NOT EXISTS pg_incremental;\n\u003C/code\u003E\u003C/pre\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EExtension\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EPurpose\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Epg_lake\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EReads Parquet files from the internal stage via SQL\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Epg_cron\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EIn-database scheduler used by \u003Ccode\u003Epg_incremental\u003C/code\u003E\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Epg_incremental\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ETracks processed files and schedules new-file callbacks\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Ch3\u003EStep 2: Create Products Table\u003C/h3\u003E\n","\u003Cp\u003EThe schema mirrors the Snowflake source table exactly, with an additional \u003Ccode\u003Esynced_at\u003C/code\u003E column to record when each row was last written by the pipeline.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 2 - Create Products Table\n-- Execute in: psql (Postgres)\nCREATE TABLE IF NOT EXISTS products (\n    product_id    INTEGER        NOT NULL PRIMARY KEY,\n    product_name  TEXT           NOT NULL,\n    category      TEXT           NOT NULL,\n    price         NUMERIC(10,2)  NOT NULL,\n    status        TEXT           NOT NULL,\n    updated_at    TIMESTAMP      NOT NULL,\n    synced_at     TIMESTAMP      NOT NULL DEFAULT now()\n);\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 3: Create Foreign Table\u003C/h3\u003E\n","\u003Cp\u003EThe foreign table covers all Parquet files written by the Snowflake task using a wildcard path. The \u003Ccode\u003Efilename 'true'\u003C/code\u003E option instructs \u003Ccode\u003Epg_lake\u003C/code\u003E to include the source file path as a \u003Ccode\u003E_filename\u003C/code\u003E column on every row, enabling the sync function to filter down to exactly one file per call.\u003C/p\u003E\n\u003Cblockquote\u003E\n","\u003Cp\u003E\u003Cstrong\u003ENote on monetary values:\u003C/strong\u003E Snowflake exports \u003Ccode\u003ENUMBER(10,2)\u003C/code\u003E columns as \u003Ccode\u003EDOUBLE PRECISION\u003C/code\u003E in Parquet format (IEEE 754 binary float). Most decimal fractions — including common prices like \u003Ccode\u003E89.99\u003C/code\u003E — cannot be represented exactly in binary floating point. While PostgreSQL's shortest-representation conversion handles most cases correctly, this is not guaranteed for all possible values. To eliminate this risk entirely, the Snowflake task exports \u003Ccode\u003Eprice\u003C/code\u003E as integer cents (\u003Ccode\u003Eprice * 100\u003C/code\u003E) and the sync function divides by 100 when inserting into Postgres. This keeps monetary values as exact integers throughout the pipeline.\u003C/p\u003E\n","\u003Cp\u003E\u003Cstrong\u003EFormat comparison for fixed-point values:\u003C/strong\u003E\u003C/p\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EFormat\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EHow \u003Ccode\u003ENUMBER(10,2)\u003C/code\u003E travels\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EPrecision risk\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003EForeign table type\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EParquet\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EExported as \u003Ccode\u003EDOUBLE\u003C/code\u003E (float64)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EYes — cents approach required\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EINTEGER\u003C/code\u003E (cents)\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ECSV\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EExported as decimal string \u003Ccode\u003E"89.99"\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ENone — exact text round-trip\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003ENUMERIC(10,2)\u003C/code\u003E directly\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EJSON\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EExported as decimal string \u003Ccode\u003E89.99\u003C/code\u003E, but many JSON parsers parse numbers as float64 internally\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EDepends on pg_lake implementation\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ETest before trusting \u003Ccode\u003ENUMERIC\u003C/code\u003E\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Cp\u003EIf you switch to CSV format, you can remove the cents conversion entirely — declare \u003Ccode\u003Eprice NUMERIC(10,2)\u003C/code\u003E in the foreign table and remove the \u003Ccode\u003Eprice_cents\u003C/code\u003E column and division from the sync function.\u003C/p\u003E\n\u003C/blockquote\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 3 - Create Foreign Table\n-- Execute in: psql (Postgres)\nCREATE FOREIGN TABLE products_stream_files (\n    product_id    INTEGER,\n    product_name  TEXT,\n    category      TEXT,\n    price_cents   INTEGER,\n    status        TEXT,\n    updated_at    TIMESTAMP,\n    _action       TEXT,\n    _is_update    BOOLEAN,\n    _filename     TEXT\n)\nSERVER pg_lake\nOPTIONS (\n    path     '@STAGE/products/product_stream_*.parquet',\n    format   'parquet',\n    filename 'true'\n);\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 4: Create Sync Function\u003C/h3\u003E\n","\u003Cp\u003EThe sync function applies a single Parquet file inside a single transaction using one set-based \u003Ccode\u003EMERGE\u003C/code\u003E — no row-by-row loop. To understand why a single statement is sufficient (and correct), you first need to understand what a Snowflake stream actually hands you.\u003C/p\u003E\n\u003Cblockquote\u003E\n","\u003Cp\u003E\u003Cstrong\u003EHow Snowflake streams work — net outcome, not history:\u003C/strong\u003E A standard (delta) stream does \u003Cstrong\u003Enot\u003C/strong\u003E replay every DML statement applied to the source. It exposes the \u003Cstrong\u003Enet change of each row between two offsets\u003C/strong\u003E (the last time the stream was consumed and now). All intermediate churn for a given key is collapsed before you ever see it. Concretely, within a single consume — and therefore within a single exported file — each \u003Ccode\u003Eproduct_id\u003C/code\u003E appears as \u003Cstrong\u003Eat most one\u003C/strong\u003E of the following:\u003C/p\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003ERecords for a key in one file\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EMETADATA$ISUPDATE\u003C/code\u003E (\u003Ccode\u003E_is_update\u003C/code\u003E)\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003ENet meaning\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EINSERT\u003C/code\u003E alone\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Efalse\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003Enet-new row\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EDELETE\u003C/code\u003E alone\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Efalse\u003C/code\u003E\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003Egenuine removal\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EDELETE\u003C/code\u003E + \u003Ccode\u003EINSERT\u003C/code\u003E pair\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Etrue\u003C/code\u003E (both)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003Eupdate\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Cp\u003EYou can never get two independent inserts, or an unrelated delete and insert, for the same key in one file. For example, if a row is deleted and then re-inserted before the next consume, the stream nets it to a single \u003Ccode\u003EINSERT\u003C/code\u003E (if the row was absent at the start offset) or to a \u003Ccode\u003EDELETE\u003C/code\u003E+\u003Ccode\u003EINSERT\u003C/code\u003E update pair (if it was present) — never to two separate events. This is why a per-key final-state resolution is always well defined.\u003C/p\u003E\n\u003C/blockquote\u003E\n","\u003Cp\u003EThe \u003Ccode\u003E_is_update\u003C/code\u003E flag is the key to routing each record correctly. The important property: a \u003Ccode\u003EDELETE\u003C/code\u003E with \u003Ccode\u003E_is_update = true\u003C/code\u003E \u003Cstrong\u003Ealways\u003C/strong\u003E has its partner \u003Ccode\u003EINSERT\u003C/code\u003E in the same file (the pair is emitted atomically in one consume). So the delete-half of an update is noise — the paired \u003Ccode\u003EINSERT\u003C/code\u003E already carries the correct final state — and only a \u003Ccode\u003EDELETE\u003C/code\u003E with \u003Ccode\u003E_is_update = false\u003C/code\u003E should actually remove a row.\u003C/p\u003E\n","\u003Cp\u003EBy filtering out the update-half deletes (\u003Ccode\u003E_action = 'DELETE' AND _is_update = true\u003C/code\u003E), the source reduces to \u003Cstrong\u003Eexactly one row per \u003Ccode\u003Eproduct_id\u003C/code\u003E\u003C/strong\u003E, each carrying a single intended action. That is precisely the shape Postgres \u003Ccode\u003EMERGE\u003C/code\u003E needs (it cannot delete and insert the same target row in one pass), so the whole file collapses into one statement:\u003C/p\u003E\n\u003Cul\u003E\u003Cli\u003E\u003Cstrong\u003Enet update\u003C/strong\u003E and \u003Cstrong\u003Ereinsert of an existing row\u003C/strong\u003E: the \u003Ccode\u003EDELETE\u003C/code\u003E is filtered out; the \u003Ccode\u003EINSERT\u003C/code\u003E matches the existing row → \u003Ccode\u003EUPDATE\u003C/code\u003E.\u003C/li\u003E\u003Cli\u003E\u003Cstrong\u003Enet insert\u003C/strong\u003E: no matching target row → \u003Ccode\u003EINSERT\u003C/code\u003E.\u003C/li\u003E\u003Cli\u003E\u003Cstrong\u003Enet delete\u003C/strong\u003E: \u003Ccode\u003EDELETE\u003C/code\u003E with \u003Ccode\u003E_is_update = false\u003C/code\u003E matches the target row → \u003Ccode\u003EDELETE\u003C/code\u003E.\u003C/li\u003E\u003C/ul\u003E\n\u003Cblockquote\u003E\n","\u003Cp\u003E\u003Cstrong\u003ENote on \u003Ccode\u003Eupdated_at\u003C/code\u003E:\u003C/strong\u003E Because routing is decided by record \u003Cem\u003Estructure\u003C/em\u003E (\u003Ccode\u003E_is_update\u003C/code\u003E) rather than by ordering, \u003Ccode\u003Eupdated_at\u003C/code\u003E is \u003Cstrong\u003Enot\u003C/strong\u003E a control key for this function — it is stored as a plain data column. Cross-file ordering is handled separately by the sorted list wrapper in Step 5 (file modification time), which sequences one file after another. Within a file there is only one net state per key, so no per-record ordering is required. Snowflake streams expose only three metadata columns — \u003Ccode\u003EMETADATA$ACTION\u003C/code\u003E, \u003Ccode\u003EMETADATA$ISUPDATE\u003C/code\u003E, and \u003Ccode\u003EMETADATA$ROW_ID\u003C/code\u003E — and \u003Ccode\u003E_is_update\u003C/code\u003E (\u003Ccode\u003EMETADATA$ISUPDATE\u003C/code\u003E) is the one that makes this deterministic.\u003C/p\u003E\n\u003C/blockquote\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 4 - Create Sync Function\n-- Execute in: psql (Postgres)\nCREATE OR REPLACE FUNCTION sync_products_from_file(filepath TEXT)\nRETURNS VOID\nLANGUAGE SQL AS $$\n    MERGE INTO products t\n    USING (\n        SELECT product_id, product_name, category, (price_cents::NUMERIC / 100)::NUMERIC(10,2) AS price,\n               status, updated_at::TIMESTAMP AS updated_at, _action\n        FROM   products_stream_files\n        WHERE  _filename = filepath\n               AND  NOT (_action = 'DELETE' AND _is_update = true)  -- drop the delete-half of updates\n    ) s\n    ON t.product_id = s.product_id\n    WHEN MATCHED AND s._action = 'DELETE' THEN\n        DELETE\n    WHEN MATCHED AND s._action = 'INSERT' THEN\n        UPDATE SET product_name = s.product_name,\n                   category     = s.category,\n                   price        = s.price,\n                   status       = s.status,\n                   updated_at   = s.updated_at,\n                   synced_at    = now()\n    WHEN NOT MATCHED AND s._action = 'INSERT' THEN\n        INSERT (product_id, product_name, category, price, status, updated_at, synced_at)\n        VALUES (s.product_id, s.product_name, s.category, s.price, s.status, s.updated_at, now());\n$$;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 5: Create Sorted List Wrapper and pg_incremental Pipeline\u003C/h3\u003E\n","\u003Cp\u003ENeither \u003Ccode\u003Epg_lake\u003C/code\u003E nor \u003Ccode\u003Epg_incremental\u003C/code\u003E sorts file results. \u003Ccode\u003Epg_incremental\u003C/code\u003E processes files in whatever order the list function returns them, and \u003Ccode\u003Elake_file.list()\u003C/code\u003E passes through the object storage API order with no sort step. To guarantee that files are applied in modification-time order, create a thin SQL wrapper that adds \u003Ccode\u003EORDER BY last_modified_time ASC\u003C/code\u003E and pass it as the \u003Ccode\u003Elist_function\u003C/code\u003E argument.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 5 - Create Sorted List Wrapper and pg_incremental Pipeline\n-- Execute in: psql (Postgres)\n\n-- Sorted wrapper: returns files ordered by modification time ascending.\n-- pg_incremental calls this with one text argument (the file pattern) and\n-- reads only the path column, so additional columns are harmless.\nCREATE OR REPLACE FUNCTION public.list_files_sorted(url_wildcard text,\n    OUT path text,\n    OUT file_size bigint,\n    OUT last_modified_time timestamptz,\n    OUT etag text)\nRETURNS SETOF record\nLANGUAGE SQL STABLE AS $$\n    SELECT path, file_size, last_modified_time, etag\n    FROM   lake_file.list(url_wildcard)\n    ORDER BY last_modified_time ASC NULLS LAST;\n$$;\n\n-- Pipeline: polls every 30 seconds, calls sync_products_from_file per new file,\n-- and processes files in modification-time order via the sorted wrapper.\nSELECT incremental.create_file_list_pipeline(\n    pipeline_name => 'products_sync_pipeline',\n    file_pattern  => '@STAGE/products/product_stream_*.parquet',\n    command       => $$SELECT sync_products_from_file($1)$$,\n    list_function => 'public.list_files_sorted',\n    schedule      => '30 seconds'\n);\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003E\u003Ccode\u003Epg_incremental\u003C/code\u003E records every successfully processed file in \u003Ccode\u003Eincremental.processed_files\u003C/code\u003E, guaranteeing exactly-once delivery even if the pipeline restarts.\u003C/p\u003E\n","\u003Ch3\u003EStep 6: Monitor the Pipeline\u003C/h3\u003E\n","\u003Cp\u003EWait 30-60 seconds for the pipeline to detect and process the initial Parquet files written by the Snowflake task.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 6 - Monitor the Pipeline\n-- Execute in: psql (Postgres)\nSELECT   jobname, start_time, end_time, status, return_message\nFROM     cron.job_run_details\n         JOIN cron.job USING (jobid)\nWHERE    jobname LIKE '%products_sync_pipeline%'\nORDER BY start_time DESC\nLIMIT    10;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 7: Verify Initial Data Loaded\u003C/h3\u003E\n","\u003Cp\u003EAll 8 products inserted in Snowflake should now be present in Postgres.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 7 - Verify Initial Data Loaded\n-- Execute in: psql (Postgres)\nSELECT   product_id, product_name, category, price, status, updated_at, synced_at\nFROM     products\nORDER BY product_id;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 8: Check Processed Files\u003C/h3\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 8 - Check Processed Files\n-- Execute in: psql (Postgres)\nSELECT *\nFROM   incremental.processed_files\nWHERE  pipeline_name = 'products_sync_pipeline';\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 9: Schedule Stage File Purge\u003C/h3\u003E\n","\u003Cp\u003E\u003Ccode\u003Eincremental.processed_files\u003C/code\u003E accumulates one row per processed file and the Parquet files themselves remain on the stage indefinitely. Create a maintenance function that purges files older than a configurable number of days and schedule it with pg_cron to run daily.\u003C/p\u003E\n","\u003Cp\u003E\u003Cstrong\u003EOperation order matters for safety.\u003C/strong\u003E If the \u003Ccode\u003Eprocessed_files\u003C/code\u003E row were deleted first and the stage delete then failed, pg_incremental would no longer recognise the file as processed and would apply it again on the next pipeline run, producing duplicate rows. To prevent this, the function deletes the stage file and flushes the cache \u003Cem\u003Ebefore\u003C/em\u003E removing the tracking row. If \u003Ccode\u003Elake_file.delete\u003C/code\u003E raises an error, the \u003Ccode\u003Eprocessed_files\u003C/code\u003E row is left intact and the file cannot be reprocessed. A per-file loop with an inner exception block isolates failures so one bad file does not abort the entire purge run — skipped files are logged as warnings.\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Postgres Setup: Step 9 - Schedule Stage File Purge\n-- Execute in: psql (Postgres)\n\nCREATE OR REPLACE FUNCTION purge_old_sync_files(older_than_days INT DEFAULT 7)\nRETURNS INT LANGUAGE plpgsql AS $$\nDECLARE\n    rec           RECORD;\n    deleted_count INT := 0;\nBEGIN\n    FOR rec IN\n        SELECT file_path\n        FROM   incremental.processed_files\n        WHERE  pipeline_name = 'products_sync_pipeline'\n          AND  processed_at  < now() - (older_than_days || ' days')::interval\n    LOOP\n        BEGIN\n            -- Stage delete first: if this fails the processed_files row is\n            -- preserved and the file will not be reprocessed\n            PERFORM lake_file.delete(rec.file_path);\n            PERFORM lake_file_cache.remove(rec.file_path);\n\n            -- Only remove the tracking row after the stage delete succeeds\n            DELETE FROM incremental.processed_files\n            WHERE  file_path = rec.file_path;\n\n            deleted_count := deleted_count + 1;\n        EXCEPTION WHEN OTHERS THEN\n            RAISE WARNING 'purge_old_sync_files: skipping % — %',\n                          rec.file_path, SQLERRM;\n        END;\n    END LOOP;\n\n    RETURN deleted_count;\nEND;\n$$;\n\n-- Run daily at 03:00 UTC, retaining the last 30 days of files\nSELECT cron.schedule(\n    'purge-old-sync-files',\n    '0 3 * * *',\n    $$SELECT purge_old_sync_files(7)$$\n);\n\n-- Confirm the job is registered\nSELECT jobid, jobname, schedule, active FROM cron.job ORDER BY jobid;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003EAdjust the \u003Ccode\u003E7\u003C/code\u003E day default in the \u003Ccode\u003Ecron.schedule\u003C/code\u003E call to match your retention requirements.\u003C/p\u003E\n<!-- ------------------------ -->\n","\u003Ch2\u003EObserve Live Sync\u003C/h2\u003E\n","\u003Cp\u003EWith both sides running, make changes in Snowflake and watch them propagate to Postgres automatically.\u003C/p\u003E\n","\u003Ch3\u003EStep 1: Make Changes in Snowflake\u003C/h3\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Observe Live Sync: Step 1 - Make Changes in Snowflake\n-- Execute in: Snowsight (Snowflake)\nUSE SCHEMA stream_lab.catalog;\n\n-- Update: raise product 1 price from 89.99 to 99.99\nUPDATE products\nSET    price = 99.99, updated_at = current_timestamp()\nWHERE  product_id = 1;\n\n-- Insert: add a new product\nINSERT INTO products (product_id, product_name, category, price, status)\nVALUES (9, 'Smart Water Bottle', 'home', 42.99, 'active');\n\n-- Delete: discontinue product 5\nDELETE FROM products WHERE product_id = 5;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EStep 2: View Pending Stream Changes\u003C/h3\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Observe Live Sync: Step 2 - View Pending Stream Changes\n-- Execute in: Snowsight (Snowflake)\nSELECT\n    METADATA$ACTION   AS _action,\n    METADATA$ISUPDATE AS _is_update,\n    product_id, product_name, category, price, status\nFROM products_stream\nORDER BY product_id;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003EYou should see 5 records: 2 for the UPDATE (DELETE old + INSERT new), 1 INSERT, and 1 DELETE.\u003C/p\u003E\n","\u003Ch3\u003EStep 3: Wait for Task to Export Changes\u003C/h3\u003E\n","\u003Cp\u003EThe task runs every minute when the stream has data. Wait up to 1 minute, then confirm the new file appeared and the stream is empty:\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Observe Live Sync: Step 3 - Wait for Task to Export Changes\n-- Execute in: Snowsight (Snowflake)\nLIST @stream_sync_stage/products/;\n\n-- Confirm stream is empty (task consumed the changes)\nSELECT count(*) AS pending_changes FROM products_stream;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003EYou should see a new Parquet file in the stage and \u003Ccode\u003Epending_changes = 0\u003C/code\u003E.\u003C/p\u003E\n","\u003Ch3\u003EStep 4: Verify Changes in Postgres\u003C/h3\u003E\n","\u003Cp\u003EWithin approximately 30 seconds of the Parquet file appearing, the \u003Ccode\u003Epg_incremental\u003C/code\u003E pipeline will detect and process it. Check that all three changes arrived:\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Observe Live Sync: Step 4 - Verify Changes in Postgres\n-- Execute in: psql (Postgres)\nSELECT   product_id, product_name, price, status, updated_at, synced_at\nFROM     products\nORDER BY product_id;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Cp\u003EExpected results:\u003C/p\u003E\n\u003Cul\u003E\u003Cli\u003EProduct 1 (\u003Ccode\u003EWireless Headphones\u003C/code\u003E) price = \u003Ccode\u003E99.99\u003C/code\u003E\u003C/li\u003E\u003Cli\u003EProduct 5 (\u003Ccode\u003EMerino Wool Socks\u003C/code\u003E) is absent\u003C/li\u003E\u003Cli\u003EProduct 9 (\u003Ccode\u003ESmart Water Bottle\u003C/code\u003E) is present\u003C/li\u003E\u003C/ul\u003E\n<!-- ------------------------ -->\n","\u003Ch2\u003ECleanup\u003C/h2\u003E\n","\u003Ch3\u003EPostgres Cleanup\u003C/h3\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Cleanup: Postgres Cleanup\n-- Execute in: psql (Postgres)\nSELECT cron.unschedule('purge-old-sync-files');\n\nDROP FUNCTION      IF EXISTS purge_old_sync_files(INT);\n\nSELECT incremental.drop_pipeline('products_sync_pipeline');\n\nDROP FUNCTION      IF EXISTS sync_products_from_file(TEXT);\nDROP FOREIGN TABLE IF EXISTS products_stream_files;\nDROP TABLE         IF EXISTS products;\n\nSELECT lake_file_cache.remove(path) FROM lake_file_cache.list();\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003ESnowflake Cleanup\u003C/h3\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Cleanup: Snowflake Cleanup\n-- Execute in: Snowsight (Snowflake)\nUSE ROLE ACCOUNTADMIN;\nUSE SCHEMA stream_lab.catalog;\n\nALTER TASK IF EXISTS stream_lab.catalog.export_products_stream_task SUSPEND;\nDROP  TASK      IF EXISTS stream_lab.catalog.export_products_stream_task;\nDROP  PROCEDURE IF EXISTS stream_lab.catalog.export_products_stream();\n\nDROP STREAM IF EXISTS stream_lab.catalog.products_stream;\nDROP TABLE  IF EXISTS stream_lab.catalog.products;\n\nREMOVE @stream_sync_stage/products/;\n\nDROP STAGE               IF EXISTS stream_lab.catalog.stream_sync_stage;\nDROP STORAGE INTEGRATION IF EXISTS PG_LAB_stream_int;\n\nDROP SCHEMA   IF EXISTS stream_lab.catalog;\nDROP DATABASE IF EXISTS stream_lab;\n\u003C/code\u003E\u003C/pre\u003E\n","\u003Ch3\u003EInfrastructure Cleanup (Optional)\u003C/h3\u003E\n","\u003Cp\u003EThe Postgres instance, network policy, and network rule are preserved by the cleanup above to avoid lengthy recreation time. To fully remove all infrastructure:\u003C/p\u003E\n\u003Cpre\u003E\u003Ccode class=\"language-sql\"\u003E-- Cleanup: Infrastructure Cleanup (Optional)\n-- Execute in: Snowsight (Snowflake)\nUSE ROLE ACCOUNTADMIN;\n\nDROP POSTGRES INSTANCE IF EXISTS PG_LAB;\nDROP NETWORK POLICY    IF EXISTS PG_LAB_network_policy;\nDROP NETWORK RULE      IF EXISTS pg_network_db.pg_network.PG_LAB_ingress_rule;\n\u003C/code\u003E\u003C/pre\u003E\n<!-- ------------------------ -->\n","\u003Ch2\u003EConclusion and Resources\u003C/h2\u003E\n","\u003Ch3\u003EWhat You Learned\u003C/h3\u003E\n","\u003Cp\u003ECongratulations! You have successfully:\u003C/p\u003E\n\u003Cul\u003E\u003Cli\u003ECreated a Snowflake Stream to capture row-level changes (INSERT, UPDATE, DELETE)\u003C/li\u003E\u003Cli\u003ECreated a Snowflake Task that exports stream records to Parquet files on a shared internal stage\u003C/li\u003E\u003Cli\u003EUnderstood stream metadata columns (\u003Ccode\u003EMETADATA$ACTION\u003C/code\u003E, \u003Ccode\u003EMETADATA$ISUPDATE\u003C/code\u003E) and how streams report the net outcome per row across INSERT, UPDATE, and DELETE\u003C/li\u003E\u003Cli\u003EWritten a Postgres sync function that applies each file with a single set-based \u003Ccode\u003EMERGE\u003C/code\u003E, using \u003Ccode\u003E_is_update\u003C/code\u003E to route each change correctly\u003C/li\u003E\u003Cli\u003EUsed \u003Ccode\u003Epg_incremental\u003C/code\u003E to automatically detect and process new stage files with exactly-once delivery\u003C/li\u003E\u003Cli\u003EObserved end-to-end sync — a change made in Snowflake appeared in Postgres within ~30 seconds\u003C/li\u003E\u003C/ul\u003E\n","\u003Ch3\u003EKey Capabilities Demonstrated\u003C/h3\u003E\n\u003Ctable\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003ESnowflake\u003C/th\u003E\u003Cth colspan=\"1\" rowspan=\"1\"\u003ESnowflake Postgres\u003C/th\u003E\u003C/tr\u003E\u003C/thead\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EStreams for change capture\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Epg_lake\u003C/code\u003E foreign tables for Parquet reads\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003ETasks for scheduled export\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Epg_incremental\u003C/code\u003E for file-list pipelines\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EInternal stage (shared with Postgres)\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003Epg_cron\u003C/code\u003E for in-database scheduling\u003C/td\u003E\u003C/tr\u003E\u003Ctr\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003E\u003Ccode\u003EINCLUDE_QUERY_ID\u003C/code\u003E for unique filenames\u003C/td\u003E\u003Ctd colspan=\"1\" rowspan=\"1\"\u003EUpsert with \u003Ccode\u003EON CONFLICT DO UPDATE\u003C/code\u003E\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\n","\u003Ch3\u003ERelated Resources\u003C/h3\u003E\n\u003Cul\u003E\u003Cli\u003E\u003Ca href=\"https://docs.snowflake.com/en/user-guide/streams-intro\"\u003ESnowflake Streams Documentation\u003C/a\u003E\u003C/li\u003E\u003Cli\u003E\u003Ca href=\"https://docs.snowflake.com/en/user-guide/tasks-intro\"\u003ESnowflake Tasks Documentation\u003C/a\u003E\u003C/li\u003E\u003Cli\u003E\u003Ca href=\"https://docs.snowflake.com/en/user-guide/snowflake-postgres/about\"\u003ESnowflake Postgres Documentation\u003C/a\u003E\u003C/li\u003E\u003Cli\u003E\u003Ca href=\"en/developers/guides/snowflake-postgres-pg-lake-iot-internal-stage/\"\u003EBidirectional Data Pipelines with pg_lake - Internal Stage\u003C/a\u003E\u003C/li\u003E\u003C/ul\u003E"],"description":"","title":"Base Quickstart CF",":type":"snowflake-site/components/contentfragment",":items":{},":itemsOrder":[],"elements":{"quickstartArticleBody":{"dataType":"string","title":"Quickstart Article Body","value":"\u003C!-- ------------------------ --\u003E\n## Overview\n\n### What You'll Build\n\nThis quickstart demonstrates how to build an automated pipeline that streams row-level changes (INSERT, UPDATE, DELETE) from a Snowflake table down to a Snowflake Postgres table. Changes made in Snowflake are exported as Parquet files to a shared internal stage by a scheduled Task, then detected and merged into Postgres by a `pg_incremental` pipeline — with no S3 bucket, no external pipeline, and no manual intervention.\n\n### What You'll Learn \n\n- How Snowflake Streams present captured changes\n- Easy way to setup change data capture from Snowflake to Postgres\n- Automate file loads with pg_incremental\n- Managing floating point numbers when exporting to parquet\n\n### What You'll Need\n\n- Snowflake account with ACCOUNTADMIN access\n- Local terminal session to run `psql`\n- `psql` client installed locally (install with `brew install postgresql` on macOS)\n- Familiarity with SQL and basic Postgres concepts\n\n### How It Works\n\nSnowflake Streams capture every row-level change made to a table since the stream was last consumed. A scheduled Snowflake Task reads the stream and writes the captured changes as Parquet files to a shared internal stage. On the Postgres side, `pg_incremental` polls the stage every 30 seconds and calls a sync function for each new file. The sync function reads the Parquet file using a `pg_lake` foreign table and merges each record into the local Postgres table.\n\n### Architecture\n\n![Architecture](https://www.snowflake.com/content/dam/snowflake-site/developers/guides/snowflake-postgres-stream-snowflake-with-pglake/architecture.png)\n\n### Stream Change Record Semantics\n\nSnowflake streams encode all three change types using two metadata columns:\n\n| `_action` | `_is_update` | Meaning | Postgres action |\n|-----------|-------------|---------|------------------|\n| `INSERT` | `FALSE` | New row inserted | INSERT |\n| `INSERT` | `TRUE` | New version of an updated row | INSERT |\n| `DELETE` | `FALSE` | Row deleted | DELETE |\n| `DELETE` | `TRUE` | Old version of an updated row | DELETE |\n\nAn UPDATE produces two stream records for the same `product_id` — a `DELETE` (old values) and an `INSERT` (new values), both with `_is_update=TRUE`. Crucially, a Snowflake stream reports the **net outcome** of each row between two consumes, not a full statement-by-statement history: within a single exported file, each `product_id` appears as at most a single `INSERT`, a single `DELETE`, or one `DELETE`+`INSERT` update pair. The sync function uses this guarantee to apply each file with a single set-based `MERGE`: the `_is_update` flag distinguishes a genuine deletion (`_is_update=FALSE`) from the delete-half of an update (`_is_update=TRUE`), so after filtering out the update-half deletes there is exactly one intended action per key. This makes UPDATE handling completely transparent — no row-by-row loop and no special update-detection logic is needed.\n\n### File Ordering Guarantee\n\nWith a scheduled task writing files over time, Postgres must apply files in the order they were written to preserve data integrity. This is handled in two layers:\n\n**Cross-run ordering** is guaranteed by one property and enforced by a sorted wrapper function:\n\n1. **Snowflake tasks run serially.** A task never executes concurrently with itself. Each run atomically consumes the full stream snapshot and writes its file before the next run can start. All files from task run N therefore have an earlier modification timestamp than any file from task run N+1.\n2. **`lake_file.list()` does not sort its results.** It passes through whatever order the object storage API returns with no sort step. `pg_incremental` also applies no sort — it builds an internal list in whatever row order the list function returns and processes files in that order. To guarantee modification-time ordering, this quickstart passes a `public.list_files_sorted` wrapper function (created in Postgres Setup Step 5) that adds `ORDER BY last_modified_time ASC`. Without this, file order would be undefined and older files could be applied after newer ones.\n\n**Within-run ordering** is handled by using `SINGLE = TRUE` in the `COPY INTO` statement. This forces all stream records from a single task run into one file. Without this, the `COPY INTO` may split records across multiple files with no guaranteed distribution order — meaning the DELETE and INSERT halves of the same UPDATE pair could land in different files and be applied out of order, producing incorrect results. With `SINGLE = TRUE`, each task run produces exactly one file containing a complete, self-consistent batch, which the sync function processes safely in a single transaction.\n\n### Data Model\n\n**Snowflake `products` table:**\n\n| Column | Type | Description |\n|--------|------|-------------|\n| `product_id` | INT | Primary key |\n| `product_name` | VARCHAR(100) | Display name |\n| `category` | VARCHAR(50) | Product category |\n| `price` | NUMBER(10,2) | Current price |\n| `status` | VARCHAR(20) | `active` or `discontinued` |\n| `updated_at` | TIMESTAMP_NTZ | Last modified in Snowflake |\n\n**Postgres `products` table** — same schema plus:\n\n| Column | Type | Description |\n|--------|------|-------------|\n| `synced_at` | TIMESTAMP | When the row was last written by the sync pipeline |\n\n### Prerequisites\n\n- Snowflake account with ACCOUNTADMIN access\n- Local terminal session to run `psql`\n- Familiarity with SQL and basic Postgres concepts\n\n\u003C!-- ------------------------ --\u003E\n## Postgres Instance Setup\n\nCreate a Snowflake Postgres instance. Skip this section if you already have an instance you want to use — just substitute its name for `PG_LAB` throughout.\n\n### Step 1: Create Network Policy\n\nSnowflake Postgres requires a network policy to allow client connections. Replace `nnn.nnn.nnn.nnn/32` with a specific IP address or CIDR for your organization.\n\n```sql\n-- Postgres Instance Setup: Step 1 - Create Network Policy\n-- Execute in: Snowsight (Snowflake)\nUSE ROLE ACCOUNTADMIN;\n\nCREATE DATABASE IF NOT EXISTS pg_network_db;\nCREATE SCHEMA  IF NOT EXISTS pg_network_db.pg_network;\n\nUSE SCHEMA pg_network_db.pg_network;\n\n-- Run this to find your current IP address if needed\nSELECT current_ip_address();\n\nCREATE OR REPLACE NETWORK RULE pg_lab_ingress_rule\n    TYPE       = IPV4\n    VALUE_LIST = ('nnn.nnn.nnn.nnn/32')\n    MODE       = POSTGRES_INGRESS\n    COMMENT    = 'Allow Postgres client connections (restrict in production)';\n\nCREATE OR REPLACE NETWORK POLICY pg_lab_network_policy\n    ALLOWED_NETWORK_RULE_LIST = ('pg_lab_ingress_rule')\n    COMMENT = 'Network policy for Snowflake Postgres instances';\n```\n\n### Step 2: Create Postgres Instance\n\n```sql\n-- Postgres Instance Setup: Step 2 - Create Postgres Instance\n-- Execute in: Snowsight (Snowflake)\n\nCREATE POSTGRES INSTANCE PG_LAB\n    AUTHENTICATION_AUTHORITY = POSTGRES_OR_SNOWFLAKE\n    COMPUTE_FAMILY           = 'STANDARD_L'\n    STORAGE_SIZE_GB          = 50\n    POSTGRES_VERSION         = 18\n    HIGH_AVAILABILITY        = FALSE\n    NETWORK_POLICY           = 'pg_lab_network_policy'\n    COMMENT                  = 'Postgres instance for stream-to-postgres quickstart';\n```\n\n\u003E **Important:** Save the username and password displayed in the `access_roles` field and the `host` value — you will need them to connect.\n\n### Step 3: Monitor Instance Status\n\nWait for the instance to reach READY state before proceeding.\n\n```sql\n-- Postgres Instance Setup: Step 3 - Monitor Instance Status\n-- Execute in: Snowsight (Snowflake)\n\nSHOW POSTGRES INSTANCES;\n```\n\n### Step 4: Configure psql Connection\n\nSet environment variables in your terminal once the instance is READY:\n\n```bash\nexport PGHOST=\u003Chostname from SHOW POSTGRES INSTANCES\u003E\nexport PGPORT=5432\nexport PGDATABASE=postgres\nexport PGUSER=snowflake_admin\nexport PGPASSWORD=\u003Cpassword from CREATE POSTGRES INSTANCE output\u003E\nexport PGSSLMODE=require\n```\n\nTest the connection:\n\n```bash\npsql -c \"SELECT version();\"\n```\n\n\u003C!-- ------------------------ --\u003E\n## Snowflake Setup\n\nCreate the Snowflake database, table, stage, stream, and export task.\n\n\u003E **Role Required:** This section requires **ACCOUNTADMIN** to create the storage integration.\n\n### Step 1: Create Database, Schema, Products Table, and Sample Data\n\nThis simulates a realistic starting point where a table already exists and contains data before the sync pipeline is set up.\n\n```sql\n-- Snowflake Setup: Step 1 - Create Database, Schema, Products Table, and Sample Data\n-- Execute in: Snowsight (Snowflake)\n\nCREATE DATABASE IF NOT EXISTS stream_lab;\nCREATE SCHEMA  IF NOT EXISTS stream_lab.catalog;\n\nUSE SCHEMA stream_lab.catalog;\n\nCREATE OR REPLACE TABLE products (\n    product_id    INT            NOT NULL,\n    product_name  VARCHAR(100)   NOT NULL,\n    category      VARCHAR(50)    NOT NULL,\n    price         NUMBER(10,2)   NOT NULL,\n    status        VARCHAR(20)    NOT NULL,\n    updated_at    TIMESTAMP_NTZ  DEFAULT current_timestamp(),  -- must be set on every INSERT and UPDATE; used for sync ordering\n    PRIMARY KEY (product_id)\n);\n\nINSERT INTO products (product_id, product_name, category, price, status)\nVALUES\n    (1, 'Wireless Headphones',   'electronics',  89.99, 'active'),\n    (2, 'USB-C Hub 7-Port',      'electronics',  49.99, 'active'),\n    (3, 'Mechanical Keyboard',   'electronics', 129.99, 'active'),\n    (4, 'Running Jacket',        'apparel',       74.99, 'active'),\n    (5, 'Merino Wool Socks',     'apparel',       18.99, 'active'),\n    (6, 'Trail Running Shoes',   'apparel',      119.99, 'active'),\n    (7, 'Bamboo Desk Organizer', 'home',          34.99, 'active'),\n    (8, 'LED Desk Lamp',         'home',          55.99, 'active');\n```\n\n### Step 2: Create Storage Integration, Stage, and Verify\n\nThe storage integration links the Snowflake internal stage to the Postgres instance's internal storage so both sides can read and write the same files.\n\n```sql\n-- Snowflake Setup: Step 2 - Create Storage Integration, Stage, and Verify\n-- Execute in: Snowsight (Snowflake)\nCREATE OR REPLACE STORAGE INTEGRATION pg_lab_stream_int\n    TYPE              = POSTGRES_INTERNAL_STORAGE\n    POSTGRES_INSTANCE = 'PG_LAB'\n    ENABLED           = TRUE;\n\nCREATE OR REPLACE STAGE stream_sync_stage\n    STORAGE_INTEGRATION = pg_lab_stream_int\n    RELATIVE_URL        = '/';\n\n-- Confirm stage is accessible (returns empty result if no files yet)\nLIST @stream_sync_stage/;\n```\n\n### Step 3: Create Stream and View Initial Rows\n\n`SHOW_INITIAL_ROWS = TRUE` causes the stream to surface all existing rows as INSERT records the first time it is consumed, regardless of when those rows were inserted. This is the recommended approach when setting up sync against a table that already contains data — no separate initial-load step is needed.\n\n`APPEND_ONLY = FALSE` captures INSERT, UPDATE, and DELETE operations going forward. Each UPDATE produces two stream records: a DELETE record (old values) and an INSERT record (new values).\n\n```sql\n-- Snowflake Setup: Step 3 - Create Stream and View Initial Rows\n-- Execute in: Snowsight (Snowflake)\nCREATE OR REPLACE STREAM products_stream\n    ON TABLE products\n    APPEND_ONLY       = FALSE\n    SHOW_INITIAL_ROWS = TRUE\n    COMMENT = 'Captures all changes to the products table for sync to Postgres';\n\n-- All 8 existing rows should appear with _action = 'INSERT', _is_update = FALSE\nSELECT\n    METADATA$ACTION   AS _action,\n    METADATA$ISUPDATE AS _is_update,\n    product_id, product_name, category, price, status\nFROM products_stream\nORDER BY product_id;\n```\n\n### Step 4: Create Export Procedure, Task, and Verify\n\n`SINGLE = TRUE` and `INCLUDE_QUERY_ID = TRUE` cannot be used together — Snowflake raises a compilation error. The solution is a stored procedure that builds a timestamp-based suffix using `EXECUTE IMMEDIATE`, so each task run writes its single file with a unique time based suffix (e.g. `products/product_stream_20240115_103000.parquet`). The task then calls the procedure instead of running `COPY INTO` directly.\n\n```sql\n-- Snowflake Setup: Step 4 - Create Export Procedure, Task, and Verify\n-- Execute in: Snowsight (Snowflake)\nCREATE OR REPLACE PROCEDURE export_products_stream()\nRETURNS STRING\nLANGUAGE SQL\nEXECUTE AS CALLER\nAS\n$$\nDECLARE\n    stage_path TEXT;\n    row_count  INTEGER;\nBEGIN\n    row_count := (SELECT COUNT(*) FROM stream_lab.catalog.products_stream);\n\n    IF (row_count = 0) THEN\n        RETURN NULL;\n    END IF;\n\n    stage_path := '@stream_sync_stage/products/product_stream_'\n                  || TO_CHAR(CURRENT_TIMESTAMP(), 'YYYYMMDD_HH24MISS')\n                  || '.parquet';\n\n    EXECUTE IMMEDIATE\n        'COPY INTO ' || :stage_path || '\n        FROM (\n            SELECT\n                product_id,\n                product_name,\n                category,\n                ROUND(price * 100)::INTEGER AS price_cents,\n                status,\n                updated_at,\n                METADATA$ACTION            AS _action,\n                METADATA$ISUPDATE::BOOLEAN AS _is_update\n            FROM stream_lab.catalog.products_stream\n        )\n        FILE_FORMAT = (TYPE = ''PARQUET'')\n        SINGLE      = TRUE';\n\n    RETURN :stage_path || ' (' || :row_count::TEXT || ' rows)';\nEND;\n$$;\n\nCREATE OR REPLACE TASK export_products_stream_task\n    WAREHOUSE = COMPUTE_WH\n    SCHEDULE  = '1 MINUTE'\nWHEN\n    SYSTEM$STREAM_HAS_DATA('stream_lab.catalog.products_stream')\nAS\n    CALL export_products_stream();\n\nALTER TASK export_products_stream_task RESUME;\n\n-- Wait up to 1 minute for the task to run on schedule, then verify\n-- Files land at products/product_stream_YYYYMMDD_HH24MISS.parquet\nLIST @stream_sync_stage/products/;\n\n-- Confirm the stream is now empty (task consumed the initial rows)\nSELECT count(*) AS pending_changes FROM products_stream;\n```\n\nWait up to 1 minute for the task to run. You should see a Parquet file containing a timestamp in the file name and `pending_changes = 0`.\n\n\u003C!-- ------------------------ --\u003E\n## Postgres Setup\n\nInstall extensions, create the products table, create the sync function, and start the `pg_incremental` pipeline.\n\n### Step 1: Enable Extensions\n\n[psql](https://www.postgresql.org/docs/current/app-psql.html) is the interactive terminal for PostgreSQL, allowing you to enter queries, execute SQL commands, and manage the database from the command line.\n\nConnect to Postgres via `psql` and enable the required extensions (assumes you set the PG* environment variables from the earlier step):\n\n```bash\npsql\n```\n\n```sql\n-- Postgres Setup: Step 1 - Enable Extensions\n-- Execute in: psql (Postgres)\nCREATE EXTENSION IF NOT EXISTS pg_lake CASCADE;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\nCREATE EXTENSION IF NOT EXISTS pg_incremental;\n```\n\n| Extension | Purpose |\n|-----------|----------|\n| `pg_lake` | Reads Parquet files from the internal stage via SQL |\n| `pg_cron` | In-database scheduler used by `pg_incremental` |\n| `pg_incremental` | Tracks processed files and schedules new-file callbacks |\n\n### Step 2: Create Products Table\n\nThe schema mirrors the Snowflake source table exactly, with an additional `synced_at` column to record when each row was last written by the pipeline.\n\n```sql\n-- Postgres Setup: Step 2 - Create Products Table\n-- Execute in: psql (Postgres)\nCREATE TABLE IF NOT EXISTS products (\n    product_id    INTEGER        NOT NULL PRIMARY KEY,\n    product_name  TEXT           NOT NULL,\n    category      TEXT           NOT NULL,\n    price         NUMERIC(10,2)  NOT NULL,\n    status        TEXT           NOT NULL,\n    updated_at    TIMESTAMP      NOT NULL,\n    synced_at     TIMESTAMP      NOT NULL DEFAULT now()\n);\n```\n\n### Step 3: Create Foreign Table\n\nThe foreign table covers all Parquet files written by the Snowflake task using a wildcard path. The `filename 'true'` option instructs `pg_lake` to include the source file path as a `_filename` column on every row, enabling the sync function to filter down to exactly one file per call.\n\n\u003E **Note on monetary values:** Snowflake exports `NUMBER(10,2)` columns as `DOUBLE PRECISION` in Parquet format (IEEE 754 binary float). Most decimal fractions — including common prices like `89.99` — cannot be represented exactly in binary floating point. While PostgreSQL's shortest-representation conversion handles most cases correctly, this is not guaranteed for all possible values. To eliminate this risk entirely, the Snowflake task exports `price` as integer cents (`price * 100`) and the sync function divides by 100 when inserting into Postgres. This keeps monetary values as exact integers throughout the pipeline.\n\u003E\n\u003E **Format comparison for fixed-point values:**\n\u003E\n\u003E | Format | How `NUMBER(10,2)` travels | Precision risk | Foreign table type |\n\u003E |--------|---------------------------|----------------|--------------------|\n\u003E | Parquet | Exported as `DOUBLE` (float64) | Yes — cents approach required | `INTEGER` (cents) |\n\u003E | CSV | Exported as decimal string `\"89.99\"` | None — exact text round-trip | `NUMERIC(10,2)` directly |\n\u003E | JSON | Exported as decimal string `89.99`, but many JSON parsers parse numbers as float64 internally | Depends on pg_lake implementation | Test before trusting `NUMERIC` |\n\u003E\n\u003E If you switch to CSV format, you can remove the cents conversion entirely — declare `price NUMERIC(10,2)` in the foreign table and remove the `price_cents` column and division from the sync function.\n\n```sql\n-- Postgres Setup: Step 3 - Create Foreign Table\n-- Execute in: psql (Postgres)\nCREATE FOREIGN TABLE products_stream_files (\n    product_id    INTEGER,\n    product_name  TEXT,\n    category      TEXT,\n    price_cents   INTEGER,\n    status        TEXT,\n    updated_at    TIMESTAMP,\n    _action       TEXT,\n    _is_update    BOOLEAN,\n    _filename     TEXT\n)\nSERVER pg_lake\nOPTIONS (\n    path     '@STAGE/products/product_stream_*.parquet',\n    format   'parquet',\n    filename 'true'\n);\n```\n\n### Step 4: Create Sync Function\n\nThe sync function applies a single Parquet file inside a single transaction using one set-based `MERGE` — no row-by-row loop. To understand why a single statement is sufficient (and correct), you first need to understand what a Snowflake stream actually hands you.\n\n\u003E **How Snowflake streams work — net outcome, not history:** A standard (delta) stream does **not** replay every DML statement applied to the source. It exposes the **net change of each row between two offsets** (the last time the stream was consumed and now). All intermediate churn for a given key is collapsed before you ever see it. Concretely, within a single consume — and therefore within a single exported file — each `product_id` appears as **at most one** of the following:\n\u003E\n\u003E | Records for a key in one file | `METADATA$ISUPDATE` (`_is_update`) | Net meaning |\n\u003E |---|---|---|\n\u003E | `INSERT` alone | `false` | net-new row |\n\u003E | `DELETE` alone | `false` | genuine removal |\n\u003E | `DELETE` + `INSERT` pair | `true` (both) | update |\n\u003E\n\u003E You can never get two independent inserts, or an unrelated delete and insert, for the same key in one file. For example, if a row is deleted and then re-inserted before the next consume, the stream nets it to a single `INSERT` (if the row was absent at the start offset) or to a `DELETE`+`INSERT` update pair (if it was present) — never to two separate events. This is why a per-key final-state resolution is always well defined.\n\nThe `_is_update` flag is the key to routing each record correctly. The important property: a `DELETE` with `_is_update = true` **always** has its partner `INSERT` in the same file (the pair is emitted atomically in one consume). So the delete-half of an update is noise — the paired `INSERT` already carries the correct final state — and only a `DELETE` with `_is_update = false` should actually remove a row.\n\nBy filtering out the update-half deletes (`_action = 'DELETE' AND _is_update = true`), the source reduces to **exactly one row per `product_id`**, each carrying a single intended action. That is precisely the shape Postgres `MERGE` needs (it cannot delete and insert the same target row in one pass), so the whole file collapses into one statement:\n\n- **net update** and **reinsert of an existing row**: the `DELETE` is filtered out; the `INSERT` matches the existing row → `UPDATE`.\n- **net insert**: no matching target row → `INSERT`.\n- **net delete**: `DELETE` with `_is_update = false` matches the target row → `DELETE`.\n\n\u003E **Note on `updated_at`:** Because routing is decided by record *structure* (`_is_update`) rather than by ordering, `updated_at` is **not** a control key for this function — it is stored as a plain data column. Cross-file ordering is handled separately by the sorted list wrapper in Step 5 (file modification time), which sequences one file after another. Within a file there is only one net state per key, so no per-record ordering is required. Snowflake streams expose only three metadata columns — `METADATA$ACTION`, `METADATA$ISUPDATE`, and `METADATA$ROW_ID` — and `_is_update` (`METADATA$ISUPDATE`) is the one that makes this deterministic.\n\n```sql\n-- Postgres Setup: Step 4 - Create Sync Function\n-- Execute in: psql (Postgres)\nCREATE OR REPLACE FUNCTION sync_products_from_file(filepath TEXT)\nRETURNS VOID\nLANGUAGE SQL AS $$\n    MERGE INTO products t\n    USING (\n        SELECT product_id, product_name, category, (price_cents::NUMERIC / 100)::NUMERIC(10,2) AS price,\n               status, updated_at::TIMESTAMP AS updated_at, _action\n        FROM   products_stream_files\n        WHERE  _filename = filepath\n               AND  NOT (_action = 'DELETE' AND _is_update = true)  -- drop the delete-half of updates\n    ) s\n    ON t.product_id = s.product_id\n    WHEN MATCHED AND s._action = 'DELETE' THEN\n        DELETE\n    WHEN MATCHED AND s._action = 'INSERT' THEN\n        UPDATE SET product_name = s.product_name,\n                   category     = s.category,\n                   price        = s.price,\n                   status       = s.status,\n                   updated_at   = s.updated_at,\n                   synced_at    = now()\n    WHEN NOT MATCHED AND s._action = 'INSERT' THEN\n        INSERT (product_id, product_name, category, price, status, updated_at, synced_at)\n        VALUES (s.product_id, s.product_name, s.category, s.price, s.status, s.updated_at, now());\n$$;\n```\n\n### Step 5: Create Sorted List Wrapper and pg_incremental Pipeline\n\nNeither `pg_lake` nor `pg_incremental` sorts file results. `pg_incremental` processes files in whatever order the list function returns them, and `lake_file.list()` passes through the object storage API order with no sort step. To guarantee that files are applied in modification-time order, create a thin SQL wrapper that adds `ORDER BY last_modified_time ASC` and pass it as the `list_function` argument.\n\n```sql\n-- Postgres Setup: Step 5 - Create Sorted List Wrapper and pg_incremental Pipeline\n-- Execute in: psql (Postgres)\n\n-- Sorted wrapper: returns files ordered by modification time ascending.\n-- pg_incremental calls this with one text argument (the file pattern) and\n-- reads only the path column, so additional columns are harmless.\nCREATE OR REPLACE FUNCTION public.list_files_sorted(url_wildcard text,\n    OUT path text,\n    OUT file_size bigint,\n    OUT last_modified_time timestamptz,\n    OUT etag text)\nRETURNS SETOF record\nLANGUAGE SQL STABLE AS $$\n    SELECT path, file_size, last_modified_time, etag\n    FROM   lake_file.list(url_wildcard)\n    ORDER BY last_modified_time ASC NULLS LAST;\n$$;\n\n-- Pipeline: polls every 30 seconds, calls sync_products_from_file per new file,\n-- and processes files in modification-time order via the sorted wrapper.\nSELECT incremental.create_file_list_pipeline(\n    pipeline_name =\u003E 'products_sync_pipeline',\n    file_pattern  =\u003E '@STAGE/products/product_stream_*.parquet',\n    command       =\u003E $$SELECT sync_products_from_file($1)$$,\n    list_function =\u003E 'public.list_files_sorted',\n    schedule      =\u003E '30 seconds'\n);\n```\n\n`pg_incremental` records every successfully processed file in `incremental.processed_files`, guaranteeing exactly-once delivery even if the pipeline restarts.\n\n### Step 6: Monitor the Pipeline\n\nWait 30-60 seconds for the pipeline to detect and process the initial Parquet files written by the Snowflake task.\n\n```sql\n-- Postgres Setup: Step 6 - Monitor the Pipeline\n-- Execute in: psql (Postgres)\nSELECT   jobname, start_time, end_time, status, return_message\nFROM     cron.job_run_details\n         JOIN cron.job USING (jobid)\nWHERE    jobname LIKE '%products_sync_pipeline%'\nORDER BY start_time DESC\nLIMIT    10;\n```\n\n### Step 7: Verify Initial Data Loaded\n\nAll 8 products inserted in Snowflake should now be present in Postgres.\n\n```sql\n-- Postgres Setup: Step 7 - Verify Initial Data Loaded\n-- Execute in: psql (Postgres)\nSELECT   product_id, product_name, category, price, status, updated_at, synced_at\nFROM     products\nORDER BY product_id;\n```\n\n### Step 8: Check Processed Files\n\n```sql\n-- Postgres Setup: Step 8 - Check Processed Files\n-- Execute in: psql (Postgres)\nSELECT *\nFROM   incremental.processed_files\nWHERE  pipeline_name = 'products_sync_pipeline';\n```\n\n### Step 9: Schedule Stage File Purge\n\n`incremental.processed_files` accumulates one row per processed file and the Parquet files themselves remain on the stage indefinitely. Create a maintenance function that purges files older than a configurable number of days and schedule it with pg_cron to run daily.\n\n**Operation order matters for safety.** If the `processed_files` row were deleted first and the stage delete then failed, pg_incremental would no longer recognise the file as processed and would apply it again on the next pipeline run, producing duplicate rows. To prevent this, the function deletes the stage file and flushes the cache *before* removing the tracking row. If `lake_file.delete` raises an error, the `processed_files` row is left intact and the file cannot be reprocessed. A per-file loop with an inner exception block isolates failures so one bad file does not abort the entire purge run — skipped files are logged as warnings.\n\n```sql\n-- Postgres Setup: Step 9 - Schedule Stage File Purge\n-- Execute in: psql (Postgres)\n\nCREATE OR REPLACE FUNCTION purge_old_sync_files(older_than_days INT DEFAULT 7)\nRETURNS INT LANGUAGE plpgsql AS $$\nDECLARE\n    rec           RECORD;\n    deleted_count INT := 0;\nBEGIN\n    FOR rec IN\n        SELECT file_path\n        FROM   incremental.processed_files\n        WHERE  pipeline_name = 'products_sync_pipeline'\n          AND  processed_at  \u003C now() - (older_than_days || ' days')::interval\n    LOOP\n        BEGIN\n            -- Stage delete first: if this fails the processed_files row is\n            -- preserved and the file will not be reprocessed\n            PERFORM lake_file.delete(rec.file_path);\n            PERFORM lake_file_cache.remove(rec.file_path);\n\n            -- Only remove the tracking row after the stage delete succeeds\n            DELETE FROM incremental.processed_files\n            WHERE  file_path = rec.file_path;\n\n            deleted_count := deleted_count + 1;\n        EXCEPTION WHEN OTHERS THEN\n            RAISE WARNING 'purge_old_sync_files: skipping % — %',\n                          rec.file_path, SQLERRM;\n        END;\n    END LOOP;\n\n    RETURN deleted_count;\nEND;\n$$;\n\n-- Run daily at 03:00 UTC, retaining the last 30 days of files\nSELECT cron.schedule(\n    'purge-old-sync-files',\n    '0 3 * * *',\n    $$SELECT purge_old_sync_files(7)$$\n);\n\n-- Confirm the job is registered\nSELECT jobid, jobname, schedule, active FROM cron.job ORDER BY jobid;\n```\n\nAdjust the `7` day default in the `cron.schedule` call to match your retention requirements.\n\n\u003C!-- ------------------------ --\u003E\n## Observe Live Sync\n\nWith both sides running, make changes in Snowflake and watch them propagate to Postgres automatically.\n\n### Step 1: Make Changes in Snowflake\n\n```sql\n-- Observe Live Sync: Step 1 - Make Changes in Snowflake\n-- Execute in: Snowsight (Snowflake)\nUSE SCHEMA stream_lab.catalog;\n\n-- Update: raise product 1 price from 89.99 to 99.99\nUPDATE products\nSET    price = 99.99, updated_at = current_timestamp()\nWHERE  product_id = 1;\n\n-- Insert: add a new product\nINSERT INTO products (product_id, product_name, category, price, status)\nVALUES (9, 'Smart Water Bottle', 'home', 42.99, 'active');\n\n-- Delete: discontinue product 5\nDELETE FROM products WHERE product_id = 5;\n```\n\n### Step 2: View Pending Stream Changes\n\n```sql\n-- Observe Live Sync: Step 2 - View Pending Stream Changes\n-- Execute in: Snowsight (Snowflake)\nSELECT\n    METADATA$ACTION   AS _action,\n    METADATA$ISUPDATE AS _is_update,\n    product_id, product_name, category, price, status\nFROM products_stream\nORDER BY product_id;\n```\n\nYou should see 5 records: 2 for the UPDATE (DELETE old + INSERT new), 1 INSERT, and 1 DELETE.\n\n### Step 3: Wait for Task to Export Changes\n\nThe task runs every minute when the stream has data. Wait up to 1 minute, then confirm the new file appeared and the stream is empty:\n\n```sql\n-- Observe Live Sync: Step 3 - Wait for Task to Export Changes\n-- Execute in: Snowsight (Snowflake)\nLIST @stream_sync_stage/products/;\n\n-- Confirm stream is empty (task consumed the changes)\nSELECT count(*) AS pending_changes FROM products_stream;\n```\n\nYou should see a new Parquet file in the stage and `pending_changes = 0`.\n\n### Step 4: Verify Changes in Postgres\n\nWithin approximately 30 seconds of the Parquet file appearing, the `pg_incremental` pipeline will detect and process it. Check that all three changes arrived:\n\n```sql\n-- Observe Live Sync: Step 4 - Verify Changes in Postgres\n-- Execute in: psql (Postgres)\nSELECT   product_id, product_name, price, status, updated_at, synced_at\nFROM     products\nORDER BY product_id;\n```\n\nExpected results:\n- Product 1 (`Wireless Headphones`) price = `99.99`\n- Product 5 (`Merino Wool Socks`) is absent\n- Product 9 (`Smart Water Bottle`) is present\n\n\u003C!-- ------------------------ --\u003E\n## Cleanup\n\n### Postgres Cleanup\n\n```sql\n-- Cleanup: Postgres Cleanup\n-- Execute in: psql (Postgres)\nSELECT cron.unschedule('purge-old-sync-files');\n\nDROP FUNCTION      IF EXISTS purge_old_sync_files(INT);\n\nSELECT incremental.drop_pipeline('products_sync_pipeline');\n\nDROP FUNCTION      IF EXISTS sync_products_from_file(TEXT);\nDROP FOREIGN TABLE IF EXISTS products_stream_files;\nDROP TABLE         IF EXISTS products;\n\nSELECT lake_file_cache.remove(path) FROM lake_file_cache.list();\n```\n\n### Snowflake Cleanup\n\n```sql\n-- Cleanup: Snowflake Cleanup\n-- Execute in: Snowsight (Snowflake)\nUSE ROLE ACCOUNTADMIN;\nUSE SCHEMA stream_lab.catalog;\n\nALTER TASK IF EXISTS stream_lab.catalog.export_products_stream_task SUSPEND;\nDROP  TASK      IF EXISTS stream_lab.catalog.export_products_stream_task;\nDROP  PROCEDURE IF EXISTS stream_lab.catalog.export_products_stream();\n\nDROP STREAM IF EXISTS stream_lab.catalog.products_stream;\nDROP TABLE  IF EXISTS stream_lab.catalog.products;\n\nREMOVE @stream_sync_stage/products/;\n\nDROP STAGE               IF EXISTS stream_lab.catalog.stream_sync_stage;\nDROP STORAGE INTEGRATION IF EXISTS PG_LAB_stream_int;\n\nDROP SCHEMA   IF EXISTS stream_lab.catalog;\nDROP DATABASE IF EXISTS stream_lab;\n```\n\n### Infrastructure Cleanup (Optional)\n\nThe Postgres instance, network policy, and network rule are preserved by the cleanup above to avoid lengthy recreation time. To fully remove all infrastructure:\n\n```sql\n-- Cleanup: Infrastructure Cleanup (Optional)\n-- Execute in: Snowsight (Snowflake)\nUSE ROLE ACCOUNTADMIN;\n\nDROP POSTGRES INSTANCE IF EXISTS PG_LAB;\nDROP NETWORK POLICY    IF EXISTS PG_LAB_network_policy;\nDROP NETWORK RULE      IF EXISTS pg_network_db.pg_network.PG_LAB_ingress_rule;\n```\n\n\u003C!-- ------------------------ --\u003E\n## Conclusion and Resources\n\n### What You Learned\n\nCongratulations! You have successfully:\n\n- Created a Snowflake Stream to capture row-level changes (INSERT, UPDATE, DELETE)\n- Created a Snowflake Task that exports stream records to Parquet files on a shared internal stage\n- Understood stream metadata columns (`METADATA$ACTION`, `METADATA$ISUPDATE`) and how streams report the net outcome per row across INSERT, UPDATE, and DELETE\n- Written a Postgres sync function that applies each file with a single set-based `MERGE`, using `_is_update` to route each change correctly\n- Used `pg_incremental` to automatically detect and process new stage files with exactly-once delivery\n- Observed end-to-end sync — a change made in Snowflake appeared in Postgres within ~30 seconds\n\n### Key Capabilities Demonstrated\n\n| Snowflake | Snowflake Postgres |\n|---|---|\n| Streams for change capture | `pg_lake` foreign tables for Parquet reads |\n| Tasks for scheduled export | `pg_incremental` for file-list pipelines |\n| Internal stage (shared with Postgres) | `pg_cron` for in-database scheduling |\n| `INCLUDE_QUERY_ID` for unique filenames | Upsert with `ON CONFLICT DO UPDATE` |\n\n### Related Resources\n\n- [Snowflake Streams Documentation](https://docs.snowflake.com/en/user-guide/streams-intro)\n- [Snowflake Tasks Documentation](https://docs.snowflake.com/en/user-guide/tasks-intro)\n- [Snowflake Postgres Documentation](https://docs.snowflake.com/en/user-guide/snowflake-postgres/about)\n- [Bidirectional Data Pipelines with pg_lake - Internal Stage](en/developers/guides/snowflake-postgres-pg-lake-iot-internal-stage/)","multiValue":false,":type":"text/x-markdown"},"quickstartArticleLogoImage":{"dataType":"string","title":"Quickstart Article Logo Image","multiValue":false,":type":"text/plain"}},"elementsOrder":["quickstartArticleBody","quickstartArticleLogoImage"],"isDeveloperGuidesPage":false,"model":"snowflake-site/models/quickstart-article"},"flexible_column_cont":{"id":"flexible-column-container-0a7ede4398","type":"2-column-75-25","alignColumns":"top","containerMaxWidth":"extra-large","topPadding":"none","bottomPadding":"none","spaceBetween":"none","reverseOnMobile":false,"carouselOnMobile":false,"backgroundImageOption":"none","flexible_column_content_container_1":{"layout":"SIMPLE","id":"container-26fcf85503",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{"quickstart_last_modi":{"id":"quickstart-last-modified-4c91f1b03b","icon":{"id":"icon","icon":"calendar",":type":"snowflake-site/components/icon","appliedCssClassNames":"snowflake-icon-blue"},"lastModifiedDatePrefix":"Updated","lastModifiedDate":"2026-08-13",":type":"snowflake-site/components/quickstart/quickstart-last-modified","appliedCssClassNames":"snowflake-responsive-component-top-padding-small"},"text":{"id":"text-f448e0f58a","additionalClasses":"qs-disclaimer-text","text":"\u003Cp\u003E\u003Cspan style=\"color: #666;\"\u003EThis content is provided as is, and is not maintained on an ongoing basis. It may be out of date with current Snowflake instances\u003C/span\u003E\u003C/p\u003E\r\n","richText":true,":type":"snowflake-site/components/text","appliedCssClassNames":"snowflake-responsive-component-top-padding-small"}},":itemsOrder":["quickstart_last_modi","text"]},"flexible_column_content_container_2":{"layout":"SIMPLE","id":"container-c5adfc402f",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{},":itemsOrder":[]},":type":"snowflake-site/components/flexible-column-container","isBlogPage":false,"isActiveTOC":false}},":itemsOrder":["contentfragment","flexible_column_cont"]},"flexible_column_content_container_2":{"layout":"SIMPLE","id":"container-d6bbdd1e94",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{"quickstart_table_of_":{"layout":"SIMPLE","id":"container-9c1d1d43f6","isDeveloperGuidesPage":false,":type":"snowflake-site/components/quickstart/quickstart-table-of-content/quickstart-table-of-content-container",":items":{"quickstart_table_of_":{"id":"quickstart-table-of-content-5241bb20cb","headings":["\u003Ch2\u003EOverview\u003C/h2\u003E","\u003Ch2\u003EPostgres Instance Setup\u003C/h2\u003E","\u003Ch2\u003ESnowflake Setup\u003C/h2\u003E","\u003Ch2\u003EPostgres Setup\u003C/h2\u003E","\u003Ch2\u003EObserve Live Sync\u003C/h2\u003E","\u003Ch2\u003ECleanup\u003C/h2\u003E","\u003Ch2\u003EConclusion and Resources\u003C/h2\u003E"],"fragmentPath":"/content/dam/snowflake-site/en/content-fragments/quickstarts/snowflake-postgres-stream-snowflake-with-pglake",":type":"snowflake-site/components/quickstart/quickstart-table-of-content"},"quickstart_button":{"id":"quickstart-button-6a2c334c9b","fragmentPath":"/content/dam/snowflake-site/en/content-fragments/quickstarts/snowflake-postgres-stream-snowflake-with-pglake",":type":"snowflake-site/components/quickstart/quickstart-button","appliedCssClassNames":"snowflake-responsive-component-top-padding-none"}},":itemsOrder":["quickstart_table_of_","quickstart_button"]}},":itemsOrder":["quickstart_table_of_"]},":type":"snowflake-site/components/flexible-column-container","isBlogPage":false,"isActiveTOC":false},"markup_editor":{"id":"markup-editor-8ab33bcae8","title":"Page CSS","cssContent":"#quickstart-template-main-flexible-container{padding:24px}#quickstart-template-main-flexible-container \u003E .snowflake-flexible-column-container-items{grid-template-columns:1fr 0}.qs-disclaimer-text p \u003E span{font-size:15px !important}@media (min-width:768px){#quickstart-template-main-flexible-container{padding:24px 32px}#quickstart-template-main-flexible-container \u003E .snowflake-flexible-column-container-items{grid-template-columns:7fr 3fr;gap:48px}}@media (max-width:767px){#quickstart-template-main-flexible-container \u003E .snowflake-flexible-column-container-items{gap:0}}@media (min-width:1024px){#quickstart-template-main-flexible-container{padding:0 92px 48px 92px}#quickstart-template-main-flexible-container \u003E .snowflake-flexible-column-container-items{gap:117px}}","isGSAPEnabled":false,":type":"snowflake-site/components/markup-editor"}},":itemsOrder":["quickstart_hero","flexible_column_cont","markup_editor"],":type":"wcm/foundation/components/responsivegrid"},"modal_container":{"layout":"SIMPLE","id":"container-6f2c57f576",":type":"snowflake-site/components/modal/modal-container",":items":{},":itemsOrder":[]},"experiencefragment-footer":{"id":"experiencefragment-70a117561b","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/footer/master/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/footer/master.xfmodel.json"},"markup_editor":{"id":"markup-editor-ba87e6ffa3","title":"Quickstarts Overrides","cssContent":".snowflake-markdown blockquote{padding:24px 32px;background:#f6f9fa;border:1px solid #29b5e8;border-radius:16px}.snowflake-markdown .snowflake-image-container img{width:auto !important;max-width:100%}.snowflake-markdown .snowflake-text ol{padding-left:20px !important}.snowflake-markdown .snowflake-text li{margin:0 0 12px 0 !important}.snowflake-markdown h3.snowflake-markdown-h3{font-size:20px !important;font-family:Texta,sans-serif !important}@media (min-width:768px){.snowflake-markdown h3.snowflake-markdown-h3{font-size:28px !important}}","isGSAPEnabled":false,":type":"snowflake-site/components/markup-editor"}},":itemsOrder":["experiencefragment-banner","experiencefragment-header","markup_editor_1950346551","responsivegrid","modal_container","experiencefragment-footer","markup_editor"],":type":"wcm/foundation/components/responsivegrid"}},":itemsOrder":["root"],"isPasswordProtected":false,"analyticsContentTags":["snowflake-site:taxonomy/product/data-engineering","snowflake-site:taxonomy/solution-center/certification/quickstart","snowflake-site:taxonomy/product/platform"],"analyticsEnabled":true,"coveoConfig":{"searchHub":"snowflake.com","organizationId":"snowflakecomputingproduction8neljofn","apiKey":"xx335921a6-2a0a-40f2-a167-e390b4766c3d","pipeline":"snowflake.com"},"analyticsDebugMode":false,"analyticsData":{"excludeFromAnalytics":false,"subCategory":"","pageType":"homepage","templateName":"quickstart-page-template","siteName":"snowflake","pageUrl":"/content/snowflake-site/global/en/developers/guides/snowflake-postgres-stream-snowflake-with-pglake","language":"en","category":"general","pageName":"Stream Snowflake Changes to Postgres with pg_lake","contentTags":["snowflake-site:taxonomy/product/data-engineering","snowflake-site:taxonomy/solution-center/certification/quickstart","snowflake-site:taxonomy/product/platform"]},":hierarchyType":"page",":path":"/content/snowflake-site/global/en/developers/guides/snowflake-postgres-stream-snowflake-with-pglake","locale":"en"}