Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tds experiment metrics #1172

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,8 @@ let package = Package(
name: "PixelExperimentKit",
dependencies: [
"PixelKit",
"BrowserServicesKit"
"BrowserServicesKit",
"Configuration"
],
resources: [
.process("Resources")
Expand Down Expand Up @@ -702,7 +703,8 @@ let package = Package(
.testTarget(
name: "PixelExperimentKitTests",
dependencies: [
"PixelExperimentKit"
"PixelExperimentKit",
"Configuration"
]
),
.testTarget(
Expand Down
14 changes: 11 additions & 3 deletions Sources/PageRefreshMonitor/PageRefreshMonitor.swift
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,11 @@ public extension PageRefreshMonitoring {
/// if three refreshes occur within a 20-second window.
public final class PageRefreshMonitor: PageRefreshMonitoring {

private let onDidDetectRefreshPattern: () -> Void
private let onDidDetectRefreshPattern: (_ numberOfRefreshes: Int) -> Void
private var store: PageRefreshStoring
private var lastRefreshedURL: URL?

public init(onDidDetectRefreshPattern: @escaping () -> Void,
public init(onDidDetectRefreshPattern: @escaping (Int) -> Void,
store: PageRefreshStoring) {
self.onDidDetectRefreshPattern = onDidDetectRefreshPattern
self.store = store
Expand All @@ -70,6 +70,14 @@ public final class PageRefreshMonitor: PageRefreshMonitoring {
public func register(for url: URL, date: Date = Date()) {
resetIfURLChanged(to: url)

// Detect a refresh pattern if two refreshes occur within 12 seconds,
// triggering the experiment pixel (sent at most once per day).
// Detecting three refreshes within 21 seconds necessarily includes detecting two within 12 seconds,
// so clearing timestamps after detecting three refreshes has no impact on pixel sending since it is sent at most once per day.
if date.timeIntervalSince(refreshTimestamps.last ?? Date.distantPast) < 12.0 {
onDidDetectRefreshPattern(2)
}

// Add the new refresh timestamp
refreshTimestamps.append(date)

Expand All @@ -78,7 +86,7 @@ public final class PageRefreshMonitor: PageRefreshMonitoring {

// Trigger detection if three refreshes occurred within 20 seconds, then reset timestamps
if refreshTimestamps.count > 2 {
onDidDetectRefreshPattern()
onDidDetectRefreshPattern(3)
NotificationCenter.default.post(name: .pageRefreshMonitorDidDetectRefreshPattern, object: self)
refreshTimestamps.removeAll() // Reset timestamps after detection
}
Expand Down
100 changes: 100 additions & 0 deletions Sources/PixelExperimentKit/TDSOverrideExperimentMetrics.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
//
// TDSOverrideExperimentMetrics.swift
//
// Copyright © 2025 DuckDuckGo. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

import Foundation
import PixelKit
import Configuration
import BrowserServicesKit

public enum TdsExperimentMetricType: String {
/// Metric triggered when the privacy toggle is used.
case privacyToggleUsed = "privacyToggleUsed"
/// Metric triggered after 2 quick refreshes.
case refresh2X = "2XRefresh"
/// Metric triggered after 3 quick refreshes.
case refresh3X = "3XRefresh"
}

public struct TDSOverrideExperimentMetrics {

public typealias FirePixelExperiment = (SubfeatureID, String, ConversionWindow, String) -> Void
public typealias FireDebugExperiment = (_ parameters: [String: String]) -> Void

private struct ExperimentConfig {
static var firePixelExperiment: FirePixelExperiment = { subfeatureID, metric, conversionWindow, value in
PixelKit.fireExperimentPixel(for: subfeatureID, metric: metric, conversionWindowDays: conversionWindow, value: value)
}
}

static func configureTDSOverrideExperimentMetrics(firePixelExperiment: @escaping FirePixelExperiment) {
ExperimentConfig.firePixelExperiment = firePixelExperiment
}

public static var activeTDSExperimentNameWithCohort: String? {
guard let featureFlagger = PixelKit.ExperimentConfig.featureFlagger else { return nil }
let activeExperiments = featureFlagger.getAllActiveExperiments()

for experimentType in TdsExperimentType.allCases {
let subfeatureID = experimentType.subfeature.rawValue
if let experimentData = activeExperiments[subfeatureID] {
return "\(subfeatureID)_\(experimentData.cohortID)"
}
}
return nil
}

public static func fireTdsExperimentMetric(
metricType: TdsExperimentMetricType,
etag: String,
fireDebugExperiment: @escaping FireDebugExperiment
) {
for experiment in TdsExperimentType.allCases {
for day in 0...5 {
ExperimentConfig.firePixelExperiment(
experiment.subfeature.rawValue,
metricType.rawValue,
day...day,
"1"
)
fireDebugBreakageExperiment(
experimentType: experiment,
etag: etag,
fire: fireDebugExperiment
)
}
}
}

private static func fireDebugBreakageExperiment(experimentType: TdsExperimentType,
etag: String,
fire: @escaping FireDebugExperiment) {
guard
let featureFlagger = PixelKit.ExperimentConfig.featureFlagger,
let experimentData = featureFlagger.getAllActiveExperiments()[experimentType.subfeature.rawValue]
else { return }

let experimentName: String = experimentType.subfeature.rawValue + experimentData.cohortID
let enrolmentDate = experimentData.enrollmentDate.toYYYYMMDDInET()
let parameters = [
"experiment": experimentName,
"enrolmentDate": enrolmentDate,
"tdsEtag": etag
]
fire(parameters)
}
}
61 changes: 47 additions & 14 deletions Tests/PageRefreshMonitorTests/PageRefreshMonitorTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -29,37 +29,46 @@ final class MockPageRefreshStore: PageRefreshStoring {
final class PageRefreshMonitorTests: XCTestCase {

var monitor: PageRefreshMonitor!
var detectionCount: Int = 0
var detectionParameters: [Int] = []

override func setUp() {
super.setUp()
monitor = PageRefreshMonitor(onDidDetectRefreshPattern: { self.detectionCount += 1 },
monitor = PageRefreshMonitor(onDidDetectRefreshPattern: { patternCount in self.detectionParameters.append(patternCount)},
store: MockPageRefreshStore())
}

// MARK: - Pattern Detection Tests

func testDoesNotDetectEventWhenRefreshesAreFewerThanThree() {
func testDoesNotDetectEventWhenRefreshesAreFewerThanTwo() {
let url = URL(string: "https://example.com/pageA")!
monitor.register(for: url)
XCTAssertEqual(detectionParameters.count, 0)
}

func testDetectsEventWhenTwoRefreshesOccurOnSameURL() {
let url = URL(string: "https://example.com/pageA")!
monitor.register(for: url)
monitor.register(for: url)
XCTAssertEqual(detectionCount, 0)
XCTAssertEqual(detectionParameters.count, 1)
XCTAssertEqual(detectionParameters, [2])
}

func testDetectsEventWhenThreeRefreshesOccurOnSameURL() {
let url = URL(string: "https://example.com/pageA")!
monitor.register(for: url)
monitor.register(for: url)
monitor.register(for: url)
XCTAssertEqual(detectionCount, 1)
XCTAssertEqual(detectionParameters.count, 3)
XCTAssertEqual(detectionParameters, [2, 2, 3])
}

func testDetectsEventTwiceWhenSixRefreshesOccurOnSameURL() {
let url = URL(string: "https://example.com/pageA")!
for _ in 1...6 {
monitor.register(for: url)
}
XCTAssertEqual(detectionCount, 2)
XCTAssertEqual(detectionParameters.count, 6)
XCTAssertEqual(detectionParameters, [2, 2, 3, 2, 2, 3])
}

// MARK: - URL Change Handling
Expand All @@ -70,7 +79,7 @@ final class PageRefreshMonitorTests: XCTestCase {
monitor.register(for: urlA)
monitor.register(for: urlB)
monitor.register(for: urlA)
XCTAssertEqual(detectionCount, 0)
XCTAssertEqual(detectionParameters.count, 0)
}

func testStartsNewCounterWhenURLChangesAndRegistersNewRefreshes() {
Expand All @@ -79,21 +88,42 @@ final class PageRefreshMonitorTests: XCTestCase {
monitor.register(for: urlA)
monitor.register(for: urlA)
monitor.register(for: urlB)
XCTAssertEqual(detectionCount, 0)
XCTAssertEqual(detectionParameters.count, 1)
XCTAssertEqual(detectionParameters, [2])
monitor.register(for: urlB)
monitor.register(for: urlB)
XCTAssertEqual(detectionCount, 1)
XCTAssertEqual(detectionParameters.count, 4)
XCTAssertEqual(detectionParameters, [2, 2, 2, 3])
}

// MARK: - Timed Pattern Detection

func testDoesNotDetectEventIfThreeRefreshesOccurAfter20Seconds() {
func testDoesNotDetectEventIfThreeRefreshesOccurAfter12Seconds() {
let url = URL(string: "https://example.com/pageA")!
let date = Date()
monitor.register(for: url, date: date)
monitor.register(for: url, date: date + 13) // 13 seconds after the first event
XCTAssertEqual(detectionParameters.count, 0)
}

func testDoesDetect2xEventIfSecondRefreshesOccurAfter12SecondsAndThirdAfter20Seconds() {
let url = URL(string: "https://example.com/pageA")!
let date = Date()
monitor.register(for: url, date: date)
monitor.register(for: url, date: date + 13) // 13 seconds after the first event
monitor.register(for: url, date: date + 21) // 21 seconds after the first event 8 seconds after second event
XCTAssertEqual(detectionParameters.count, 1)
XCTAssertEqual(detectionParameters, [2])
}

func testDoesDetect2xEventIfThreeRefreshesOccurAfter20Seconds() {
let url = URL(string: "https://example.com/pageA")!
let date = Date()
monitor.register(for: url, date: date)
monitor.register(for: url, date: date)
monitor.register(for: url, date: date + 21) // 21 seconds after the first event
XCTAssertEqual(detectionCount, 0)
XCTAssertEqual(detectionParameters.count, 1)
XCTAssertEqual(detectionParameters, [2])
}

func testDetectsEventIfThreeRefreshesOccurWithin20Seconds() {
Expand All @@ -102,7 +132,8 @@ final class PageRefreshMonitorTests: XCTestCase {
monitor.register(for: url, date: date)
monitor.register(for: url, date: date)
monitor.register(for: url, date: date + 19) // 19 seconds after the first event
XCTAssertEqual(detectionCount, 1)
XCTAssertEqual(detectionParameters.count, 2)
XCTAssertEqual(detectionParameters, [2, 3])
}

func testDetectsEventIfRefreshesAreWithinOverall20SecondWindow() {
Expand All @@ -111,9 +142,11 @@ final class PageRefreshMonitorTests: XCTestCase {
monitor.register(for: url, date: date)
monitor.register(for: url, date: date + 19) // 19 seconds after the first event
monitor.register(for: url, date: date + 21) // 21 seconds after the first event (2 seconds after second event)
XCTAssertEqual(detectionCount, 0)
XCTAssertEqual(detectionParameters.count, 1)
XCTAssertEqual(detectionParameters, [2])
monitor.register(for: url, date: date + 23) // 23 seconds after the first event (4 seconds after second event)
XCTAssertEqual(detectionCount, 1)
XCTAssertEqual(detectionParameters.count, 3)
XCTAssertEqual(detectionParameters, [2, 2, 3])
}

}
101 changes: 101 additions & 0 deletions Tests/PixelExperimentKitTests/TDSOverrideExperimentMetricsTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
//
// TDSOverrideExperimentMetricsTests.swift
//
// Copyright © 2025 DuckDuckGo. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

import XCTest
@testable import PixelExperimentKit
import BrowserServicesKit
import Configuration
import PixelKit

final class TDSOverrideExperimentMetricsTests: XCTestCase {

var mockFeatureFlagger: MockFeatureFlagger!
var pixelCalls: [(SubfeatureID, String, ClosedRange<Int>, String)] = []
var debugCalls: [[String: String]] = []

override func setUpWithError() throws {
mockFeatureFlagger = MockFeatureFlagger()
PixelKit.configureExperimentKit(featureFlagger: mockFeatureFlagger, eventTracker: ExperimentEventTracker(store: MockExperimentActionPixelStore()), fire: { _, _, _ in })
TDSOverrideExperimentMetrics.configureTDSOverrideExperimentMetrics { subfeatureID, metric, conversionWindow, value in
self.pixelCalls.append((subfeatureID, metric, conversionWindow, value))
}
}

override func tearDownWithError() throws {
mockFeatureFlagger = nil
}

func test_OnfireTdsExperimentMetricPrivacyToggleUsed_WhenExperimentActive_ThenCorrectPixelFunctionsCalled() {
// GIVEN
mockFeatureFlagger.experiments = [
TdsExperimentType.allCases[3].subfeature.rawValue: ExperimentData(parentID: "someParentID", cohortID: "testCohort", enrollmentDate: Date())
]

// WHEN
TDSOverrideExperimentMetrics.fireTdsExperimentMetric(metricType: .privacyToggleUsed, etag: "testEtag") { parameters in
self.debugCalls.append(parameters)
}

// THEN
XCTAssertEqual(pixelCalls.count, TdsExperimentType.allCases.count * 6, "firePixelExperiment should be called for each experiment and each conversionWindow 0...5.")
XCTAssertEqual(pixelCalls.first?.0, TdsExperimentType.allCases[0].subfeature.rawValue, "expected SubfeatureID should be passed as parameter")
XCTAssertEqual(pixelCalls.first?.1, "privacyToggleUsed", "expected metric should be passed as parameter")
XCTAssertEqual(pixelCalls.first?.2, 0...0, "expected Conversion Window should be passed as parameter")
XCTAssertEqual(pixelCalls.first?.3, "1", "expected Value should be passed as parameter")
XCTAssertEqual(debugCalls.count, 6, "fireDebugExperiment should be called for each conversionWindow on one experiment.")
XCTAssertEqual(debugCalls.first?["tdsEtag"], "testEtag")
XCTAssertEqual(debugCalls.first?["experiment"], "\(TdsExperimentType.allCases[3].experiment.rawValue)testCohort")
}

func test_OnfireTdsExperimentMetricPrivacyToggleUsed_WhenNoExperimentActive_ThenCorrectPixelFunctionsCalled() {
// WHEN
TDSOverrideExperimentMetrics.fireTdsExperimentMetric(metricType: .privacyToggleUsed, etag: "testEtag") { parameters in
self.debugCalls.append(parameters)
}

// THEN
XCTAssertEqual(pixelCalls.count, TdsExperimentType.allCases.count * 6, "firePixelExperiment should be called for each experiment and each conversionWindow 0...5.")
XCTAssertEqual(pixelCalls.first?.0, TdsExperimentType.allCases[0].subfeature.rawValue, "expected SubfeatureID should be passed as parameter")
XCTAssertEqual(pixelCalls.first?.1, "privacyToggleUsed", "expected metric should be passed as parameter")
XCTAssertEqual(pixelCalls.first?.2, 0...0, "expected Conversion Window should be passed as parameter")
XCTAssertEqual(pixelCalls.first?.3, "1", "expected Value should be passed as parameter")
XCTAssertTrue(debugCalls.isEmpty)
}

func test_OnGetActiveTDSExperimentNameWithCohort_WhenExperimentActive_ThenCorrectExperimentNameReturned() {
// GIVEN
mockFeatureFlagger.experiments = [
TdsExperimentType.allCases[3].subfeature.rawValue: ExperimentData(parentID: "someParentID", cohortID: "testCohort", enrollmentDate: Date())
]

// WHEN
let experimentName = TDSOverrideExperimentMetrics.activeTDSExperimentNameWithCohort

// THEN
XCTAssertEqual(experimentName, "\(TdsExperimentType.allCases[3].subfeature.rawValue)_testCohort")
}

func test_OnGetActiveTDSExperimentNameWithCohort_WhenNoExperimentActive_ThenCorrectExperimentNameReturned() {
// WHEN
let experimentName = TDSOverrideExperimentMetrics.activeTDSExperimentNameWithCohort

// THEN
XCTAssertNil(experimentName)
}

}
Loading